2 * Copyright (c) 2012 Chelsio Communications, Inc.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
32 #include "opt_inet6.h"
35 #include <sys/param.h>
36 #include <sys/types.h>
37 #include <sys/kernel.h>
39 #include <sys/module.h>
40 #include <sys/protosw.h>
41 #include <sys/refcount.h>
42 #include <sys/domain.h>
43 #include <sys/fnv_hash.h>
44 #include <sys/socket.h>
45 #include <sys/socketvar.h>
46 #include <net/ethernet.h>
48 #include <net/if_types.h>
49 #include <net/if_vlan_var.h>
50 #include <net/route.h>
51 #include <netinet/in.h>
52 #include <netinet/in_fib.h>
53 #include <netinet/in_pcb.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip6.h>
56 #include <netinet6/in6_fib.h>
57 #include <netinet6/scope6_var.h>
58 #include <netinet/tcp_timer.h>
60 #include <netinet/tcp_fsm.h>
61 #include <netinet/tcp_var.h>
62 #include <netinet/toecore.h>
64 #include "common/common.h"
65 #include "common/t4_msg.h"
66 #include "common/t4_regs.h"
67 #include "tom/t4_tom_l2t.h"
68 #include "tom/t4_tom.h"
71 static int alloc_stid(struct adapter *, struct listen_ctx *, int);
72 static struct listen_ctx *lookup_stid(struct adapter *, int);
73 static void free_stid(struct adapter *, struct listen_ctx *);
76 static struct listen_ctx *alloc_lctx(struct adapter *, struct inpcb *,
78 static int free_lctx(struct adapter *, struct listen_ctx *);
79 static void hold_lctx(struct listen_ctx *);
80 static void listen_hash_add(struct adapter *, struct listen_ctx *);
81 static struct listen_ctx *listen_hash_find(struct adapter *, struct inpcb *);
82 static struct listen_ctx *listen_hash_del(struct adapter *, struct inpcb *);
83 static struct inpcb *release_lctx(struct adapter *, struct listen_ctx *);
85 static inline void save_qids_in_mbuf(struct mbuf *, struct vi_info *);
86 static inline void get_qids_from_mbuf(struct mbuf *m, int *, int *);
87 static void send_reset_synqe(struct toedev *, struct synq_entry *);
90 alloc_stid(struct adapter *sc, struct listen_ctx *lctx, int isipv6)
92 struct tid_info *t = &sc->tids;
93 u_int stid, n, f, mask;
94 struct stid_region *sr = &lctx->stid_region;
97 * An IPv6 server needs 2 naturally aligned stids (1 stid = 4 cells) in
98 * the TCAM. The start of the stid region is properly aligned (the chip
99 * requires each region to be 128-cell aligned).
103 KASSERT((t->stid_base & mask) == 0 && (t->nstids & mask) == 0,
104 ("%s: stid region (%u, %u) not properly aligned. n = %u",
105 __func__, t->stid_base, t->nstids, n));
107 mtx_lock(&t->stid_lock);
108 if (n > t->nstids - t->stids_in_use) {
109 mtx_unlock(&t->stid_lock);
113 if (t->nstids_free_head >= n) {
115 * This allocation will definitely succeed because the region
116 * starts at a good alignment and we just checked we have enough
119 f = t->nstids_free_head & mask;
120 t->nstids_free_head -= n + f;
121 stid = t->nstids_free_head;
122 TAILQ_INSERT_HEAD(&t->stids, sr, link);
124 struct stid_region *s;
126 stid = t->nstids_free_head;
127 TAILQ_FOREACH(s, &t->stids, link) {
128 stid += s->used + s->free;
130 if (s->free >= n + f) {
133 TAILQ_INSERT_AFTER(&t->stids, s, sr, link);
138 if (__predict_false(stid != t->nstids)) {
139 panic("%s: stids TAILQ (%p) corrupt."
140 " At %d instead of %d at the end of the queue.",
141 __func__, &t->stids, stid, t->nstids);
144 mtx_unlock(&t->stid_lock);
151 t->stids_in_use += n;
152 t->stid_tab[stid] = lctx;
153 mtx_unlock(&t->stid_lock);
155 KASSERT(((stid + t->stid_base) & mask) == 0,
156 ("%s: EDOOFUS.", __func__));
157 return (stid + t->stid_base);
160 static struct listen_ctx *
161 lookup_stid(struct adapter *sc, int stid)
163 struct tid_info *t = &sc->tids;
165 return (t->stid_tab[stid - t->stid_base]);
169 free_stid(struct adapter *sc, struct listen_ctx *lctx)
171 struct tid_info *t = &sc->tids;
172 struct stid_region *sr = &lctx->stid_region;
173 struct stid_region *s;
175 KASSERT(sr->used > 0, ("%s: nonsense free (%d)", __func__, sr->used));
177 mtx_lock(&t->stid_lock);
178 s = TAILQ_PREV(sr, stid_head, link);
180 s->free += sr->used + sr->free;
182 t->nstids_free_head += sr->used + sr->free;
183 KASSERT(t->stids_in_use >= sr->used,
184 ("%s: stids_in_use (%u) < stids being freed (%u)", __func__,
185 t->stids_in_use, sr->used));
186 t->stids_in_use -= sr->used;
187 TAILQ_REMOVE(&t->stids, sr, link);
188 mtx_unlock(&t->stid_lock);
191 static struct listen_ctx *
192 alloc_lctx(struct adapter *sc, struct inpcb *inp, struct vi_info *vi)
194 struct listen_ctx *lctx;
196 INP_WLOCK_ASSERT(inp);
198 lctx = malloc(sizeof(struct listen_ctx), M_CXGBE, M_NOWAIT | M_ZERO);
202 lctx->stid = alloc_stid(sc, lctx, inp->inp_vflag & INP_IPV6);
203 if (lctx->stid < 0) {
208 if (inp->inp_vflag & INP_IPV6 &&
209 !IN6_ARE_ADDR_EQUAL(&in6addr_any, &inp->in6p_laddr)) {
210 struct tom_data *td = sc->tom_softc;
212 lctx->ce = hold_lip(td, &inp->in6p_laddr, NULL);
213 if (lctx->ce == NULL) {
219 lctx->ctrlq = &sc->sge.ctrlq[vi->pi->port_id];
220 lctx->ofld_rxq = &sc->sge.ofld_rxq[vi->first_ofld_rxq];
221 refcount_init(&lctx->refcount, 1);
222 TAILQ_INIT(&lctx->synq);
225 lctx->vnet = inp->inp_socket->so_vnet;
231 /* Don't call this directly, use release_lctx instead */
233 free_lctx(struct adapter *sc, struct listen_ctx *lctx)
235 struct inpcb *inp = lctx->inp;
236 struct tom_data *td = sc->tom_softc;
238 INP_WLOCK_ASSERT(inp);
239 KASSERT(lctx->refcount == 0,
240 ("%s: refcount %d", __func__, lctx->refcount));
241 KASSERT(TAILQ_EMPTY(&lctx->synq),
242 ("%s: synq not empty.", __func__));
243 KASSERT(lctx->stid >= 0, ("%s: bad stid %d.", __func__, lctx->stid));
245 CTR4(KTR_CXGBE, "%s: stid %u, lctx %p, inp %p",
246 __func__, lctx->stid, lctx, lctx->inp);
249 release_lip(td, lctx->ce);
253 return (in_pcbrele_wlocked(inp));
257 hold_lctx(struct listen_ctx *lctx)
260 refcount_acquire(&lctx->refcount);
263 static inline uint32_t
264 listen_hashfn(void *key, u_long mask)
267 return (fnv_32_buf(&key, sizeof(key), FNV1_32_INIT) & mask);
271 * Add a listen_ctx entry to the listen hash table.
274 listen_hash_add(struct adapter *sc, struct listen_ctx *lctx)
276 struct tom_data *td = sc->tom_softc;
277 int bucket = listen_hashfn(lctx->inp, td->listen_mask);
279 mtx_lock(&td->lctx_hash_lock);
280 LIST_INSERT_HEAD(&td->listen_hash[bucket], lctx, link);
282 mtx_unlock(&td->lctx_hash_lock);
286 * Look for the listening socket's context entry in the hash and return it.
288 static struct listen_ctx *
289 listen_hash_find(struct adapter *sc, struct inpcb *inp)
291 struct tom_data *td = sc->tom_softc;
292 int bucket = listen_hashfn(inp, td->listen_mask);
293 struct listen_ctx *lctx;
295 mtx_lock(&td->lctx_hash_lock);
296 LIST_FOREACH(lctx, &td->listen_hash[bucket], link) {
297 if (lctx->inp == inp)
300 mtx_unlock(&td->lctx_hash_lock);
306 * Removes the listen_ctx structure for inp from the hash and returns it.
308 static struct listen_ctx *
309 listen_hash_del(struct adapter *sc, struct inpcb *inp)
311 struct tom_data *td = sc->tom_softc;
312 int bucket = listen_hashfn(inp, td->listen_mask);
313 struct listen_ctx *lctx, *l;
315 mtx_lock(&td->lctx_hash_lock);
316 LIST_FOREACH_SAFE(lctx, &td->listen_hash[bucket], link, l) {
317 if (lctx->inp == inp) {
318 LIST_REMOVE(lctx, link);
323 mtx_unlock(&td->lctx_hash_lock);
329 * Releases a hold on the lctx. Must be called with the listening socket's inp
330 * locked. The inp may be freed by this function and it returns NULL to
333 static struct inpcb *
334 release_lctx(struct adapter *sc, struct listen_ctx *lctx)
336 struct inpcb *inp = lctx->inp;
339 INP_WLOCK_ASSERT(inp);
340 if (refcount_release(&lctx->refcount))
341 inp_freed = free_lctx(sc, lctx);
343 return (inp_freed ? NULL : inp);
347 send_reset_synqe(struct toedev *tod, struct synq_entry *synqe)
349 struct adapter *sc = tod->tod_softc;
350 struct mbuf *m = synqe->syn;
351 struct ifnet *ifp = m->m_pkthdr.rcvif;
352 struct vi_info *vi = ifp->if_softc;
353 struct port_info *pi = vi->pi;
354 struct l2t_entry *e = &sc->l2t->l2tab[synqe->l2e_idx];
356 struct fw_flowc_wr *flowc;
357 struct cpl_abort_req *req;
358 int txqid, rxqid, flowclen;
359 struct sge_wrq *ofld_txq;
360 struct sge_ofld_rxq *ofld_rxq;
361 const int nparams = 6;
362 unsigned int pfvf = G_FW_VIID_PFN(vi->viid) << S_FW_VIID_PFN;
364 INP_WLOCK_ASSERT(synqe->lctx->inp);
366 CTR5(KTR_CXGBE, "%s: synqe %p (0x%x), tid %d%s",
367 __func__, synqe, synqe->flags, synqe->tid,
368 synqe->flags & TPF_ABORT_SHUTDOWN ?
369 " (abort already in progress)" : "");
370 if (synqe->flags & TPF_ABORT_SHUTDOWN)
371 return; /* abort already in progress */
372 synqe->flags |= TPF_ABORT_SHUTDOWN;
374 get_qids_from_mbuf(m, &txqid, &rxqid);
375 ofld_txq = &sc->sge.ofld_txq[txqid];
376 ofld_rxq = &sc->sge.ofld_rxq[rxqid];
378 /* The wrqe will have two WRs - a flowc followed by an abort_req */
379 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
381 wr = alloc_wrqe(roundup2(flowclen, EQ_ESIZE) + sizeof(*req), ofld_txq);
384 panic("%s: allocation failure.", __func__);
387 req = (void *)((caddr_t)flowc + roundup2(flowclen, EQ_ESIZE));
389 /* First the flowc ... */
390 memset(flowc, 0, wr->wr_len);
391 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
392 V_FW_FLOWC_WR_NPARAMS(nparams));
393 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
394 V_FW_WR_FLOWID(synqe->tid));
395 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
396 flowc->mnemval[0].val = htobe32(pfvf);
397 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
398 flowc->mnemval[1].val = htobe32(pi->tx_chan);
399 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
400 flowc->mnemval[2].val = htobe32(pi->tx_chan);
401 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
402 flowc->mnemval[3].val = htobe32(ofld_rxq->iq.abs_id);
403 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
404 flowc->mnemval[4].val = htobe32(512);
405 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS;
406 flowc->mnemval[5].val = htobe32(512);
407 synqe->flags |= TPF_FLOWC_WR_SENT;
409 /* ... then ABORT request */
410 INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, synqe->tid);
411 req->rsvd0 = 0; /* don't have a snd_nxt */
412 req->rsvd1 = 1; /* no data sent yet */
413 req->cmd = CPL_ABORT_SEND_RST;
415 t4_l2t_send(sc, wr, e);
419 create_server(struct adapter *sc, struct listen_ctx *lctx)
422 struct cpl_pass_open_req *req;
423 struct inpcb *inp = lctx->inp;
425 wr = alloc_wrqe(sizeof(*req), lctx->ctrlq);
427 log(LOG_ERR, "%s: allocation failure", __func__);
433 OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, lctx->stid));
434 req->local_port = inp->inp_lport;
436 req->local_ip = inp->inp_laddr.s_addr;
438 req->opt0 = htobe64(V_TX_CHAN(lctx->ctrlq->eq.tx_chan));
439 req->opt1 = htobe64(V_CONN_POLICY(CPL_CONN_POLICY_ASK) |
440 F_SYN_RSS_ENABLE | V_SYN_RSS_QUEUE(lctx->ofld_rxq->iq.abs_id));
447 create_server6(struct adapter *sc, struct listen_ctx *lctx)
450 struct cpl_pass_open_req6 *req;
451 struct inpcb *inp = lctx->inp;
453 wr = alloc_wrqe(sizeof(*req), lctx->ctrlq);
455 log(LOG_ERR, "%s: allocation failure", __func__);
461 OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, lctx->stid));
462 req->local_port = inp->inp_lport;
464 req->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
465 req->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
468 req->opt0 = htobe64(V_TX_CHAN(lctx->ctrlq->eq.tx_chan));
469 req->opt1 = htobe64(V_CONN_POLICY(CPL_CONN_POLICY_ASK) |
470 F_SYN_RSS_ENABLE | V_SYN_RSS_QUEUE(lctx->ofld_rxq->iq.abs_id));
477 destroy_server(struct adapter *sc, struct listen_ctx *lctx)
480 struct cpl_close_listsvr_req *req;
482 wr = alloc_wrqe(sizeof(*req), lctx->ctrlq);
485 panic("%s: allocation failure.", __func__);
490 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ,
492 req->reply_ctrl = htobe16(lctx->ofld_rxq->iq.abs_id);
493 req->rsvd = htobe16(0);
500 * Start a listening server by sending a passive open request to HW.
502 * Can't take adapter lock here and access to sc->flags,
503 * sc->offload_map, if_capenable are all race prone.
506 t4_listen_start(struct toedev *tod, struct tcpcb *tp)
508 struct adapter *sc = tod->tod_softc;
510 struct port_info *pi;
511 struct inpcb *inp = tp->t_inpcb;
512 struct listen_ctx *lctx;
515 INP_WLOCK_ASSERT(inp);
517 /* Don't start a hardware listener for any loopback address. */
518 if (inp->inp_vflag & INP_IPV6 && IN6_IS_ADDR_LOOPBACK(&inp->in6p_laddr))
520 if (!(inp->inp_vflag & INP_IPV6) &&
521 IN_LOOPBACK(ntohl(inp->inp_laddr.s_addr)))
526 log(LOG_ERR, "%s: listen request ignored, %s is busy",
527 __func__, device_get_nameunit(sc->dev));
531 KASSERT(uld_active(sc, ULD_TOM),
532 ("%s: TOM not initialized", __func__));
536 * Find an initialized VI with IFCAP_TOE (4 or 6). We'll use the first
537 * such VI's queues to send the passive open and receive the reply to
540 * XXX: need a way to mark a port in use by offload. if_cxgbe should
541 * then reject any attempt to bring down such a port (and maybe reject
542 * attempts to disable IFCAP_TOE on that port too?).
544 for_each_port(sc, i) {
546 for_each_vi(pi, v, vi) {
547 if (vi->flags & VI_INIT_DONE &&
548 vi->ifp->if_capenable & IFCAP_TOE)
552 goto done; /* no port that's UP with IFCAP_TOE enabled */
555 if (listen_hash_find(sc, inp) != NULL)
556 goto done; /* already setup */
558 lctx = alloc_lctx(sc, inp, vi);
561 "%s: listen request ignored, %s couldn't allocate lctx\n",
562 __func__, device_get_nameunit(sc->dev));
565 listen_hash_add(sc, lctx);
567 CTR6(KTR_CXGBE, "%s: stid %u (%s), lctx %p, inp %p vflag 0x%x",
568 __func__, lctx->stid, tcpstates[tp->t_state], lctx, inp,
571 if (inp->inp_vflag & INP_IPV6)
572 rc = create_server6(sc, lctx);
574 rc = create_server(sc, lctx);
576 log(LOG_ERR, "%s: %s failed to create hw listener: %d.\n",
577 __func__, device_get_nameunit(sc->dev), rc);
578 (void) listen_hash_del(sc, inp);
579 inp = release_lctx(sc, lctx);
580 /* can't be freed, host stack has a reference */
581 KASSERT(inp != NULL, ("%s: inp freed", __func__));
584 lctx->flags |= LCTX_RPL_PENDING;
593 t4_listen_stop(struct toedev *tod, struct tcpcb *tp)
595 struct listen_ctx *lctx;
596 struct adapter *sc = tod->tod_softc;
597 struct inpcb *inp = tp->t_inpcb;
598 struct synq_entry *synqe;
600 INP_WLOCK_ASSERT(inp);
602 lctx = listen_hash_del(sc, inp);
604 return (ENOENT); /* no hardware listener for this inp */
606 CTR4(KTR_CXGBE, "%s: stid %u, lctx %p, flags %x", __func__, lctx->stid,
610 * If the reply to the PASS_OPEN is still pending we'll wait for it to
611 * arrive and clean up when it does.
613 if (lctx->flags & LCTX_RPL_PENDING) {
614 KASSERT(TAILQ_EMPTY(&lctx->synq),
615 ("%s: synq not empty.", __func__));
616 return (EINPROGRESS);
620 * The host stack will abort all the connections on the listening
621 * socket's so_comp. It doesn't know about the connections on the synq
622 * so we need to take care of those.
624 TAILQ_FOREACH(synqe, &lctx->synq, link) {
625 if (synqe->flags & TPF_SYNQE_HAS_L2TE)
626 send_reset_synqe(tod, synqe);
629 destroy_server(sc, lctx);
634 hold_synqe(struct synq_entry *synqe)
637 refcount_acquire(&synqe->refcnt);
641 release_synqe(struct synq_entry *synqe)
644 if (refcount_release(&synqe->refcnt)) {
645 int needfree = synqe->flags & TPF_SYNQE_NEEDFREE;
649 free(synqe, M_CXGBE);
654 t4_syncache_added(struct toedev *tod __unused, void *arg)
656 struct synq_entry *synqe = arg;
662 t4_syncache_removed(struct toedev *tod __unused, void *arg)
664 struct synq_entry *synqe = arg;
666 release_synqe(synqe);
670 t4_syncache_respond(struct toedev *tod, void *arg, struct mbuf *m)
672 struct adapter *sc = tod->tod_softc;
673 struct synq_entry *synqe = arg;
677 struct ip *ip = mtod(m, struct ip *);
680 wr = (struct wrqe *)atomic_readandclear_ptr(&synqe->wr);
686 if (ip->ip_v == IPVERSION)
687 th = (void *)(ip + 1);
689 th = (void *)((struct ip6_hdr *)ip + 1);
690 bzero(&to, sizeof(to));
691 tcp_dooptions(&to, (void *)(th + 1), (th->th_off << 2) - sizeof(*th),
694 /* save these for later */
695 synqe->iss = be32toh(th->th_seq);
696 synqe->ts = to.to_tsval;
698 if (chip_id(sc) >= CHELSIO_T5) {
699 struct cpl_t5_pass_accept_rpl *rpl5 = wrtod(wr);
701 rpl5->iss = th->th_seq;
704 e = &sc->l2t->l2tab[synqe->l2e_idx];
705 t4_l2t_send(sc, wr, e);
707 m_freem(m); /* don't need this any more */
712 do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
715 struct adapter *sc = iq->adapter;
716 const struct cpl_pass_open_rpl *cpl = (const void *)(rss + 1);
717 int stid = GET_TID(cpl);
718 unsigned int status = cpl->status;
719 struct listen_ctx *lctx = lookup_stid(sc, stid);
720 struct inpcb *inp = lctx->inp;
722 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
725 KASSERT(opcode == CPL_PASS_OPEN_RPL,
726 ("%s: unexpected opcode 0x%x", __func__, opcode));
727 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
728 KASSERT(lctx->stid == stid, ("%s: lctx stid mismatch", __func__));
732 CTR4(KTR_CXGBE, "%s: stid %d, status %u, flags 0x%x",
733 __func__, stid, status, lctx->flags);
735 lctx->flags &= ~LCTX_RPL_PENDING;
737 if (status != CPL_ERR_NONE)
738 log(LOG_ERR, "listener (stid %u) failed: %d\n", stid, status);
742 * If the inp has been dropped (listening socket closed) then
743 * listen_stop must have run and taken the inp out of the hash.
745 if (inp->inp_flags & INP_DROPPED) {
746 KASSERT(listen_hash_del(sc, inp) == NULL,
747 ("%s: inp %p still in listen hash", __func__, inp));
751 if (inp->inp_flags & INP_DROPPED && status != CPL_ERR_NONE) {
752 if (release_lctx(sc, lctx) != NULL)
758 * Listening socket stopped listening earlier and now the chip tells us
759 * it has started the hardware listener. Stop it; the lctx will be
760 * released in do_close_server_rpl.
762 if (inp->inp_flags & INP_DROPPED) {
763 destroy_server(sc, lctx);
769 * Failed to start hardware listener. Take inp out of the hash and
770 * release our reference on it. An error message has been logged
773 if (status != CPL_ERR_NONE) {
774 listen_hash_del(sc, inp);
775 if (release_lctx(sc, lctx) != NULL)
780 /* hardware listener open for business */
787 do_close_server_rpl(struct sge_iq *iq, const struct rss_header *rss,
790 struct adapter *sc = iq->adapter;
791 const struct cpl_close_listsvr_rpl *cpl = (const void *)(rss + 1);
792 int stid = GET_TID(cpl);
793 unsigned int status = cpl->status;
794 struct listen_ctx *lctx = lookup_stid(sc, stid);
795 struct inpcb *inp = lctx->inp;
797 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
800 KASSERT(opcode == CPL_CLOSE_LISTSRV_RPL,
801 ("%s: unexpected opcode 0x%x", __func__, opcode));
802 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
803 KASSERT(lctx->stid == stid, ("%s: lctx stid mismatch", __func__));
805 CTR3(KTR_CXGBE, "%s: stid %u, status %u", __func__, stid, status);
807 if (status != CPL_ERR_NONE) {
808 log(LOG_ERR, "%s: failed (%u) to close listener for stid %u\n",
809 __func__, status, stid);
814 inp = release_lctx(sc, lctx);
822 done_with_synqe(struct adapter *sc, struct synq_entry *synqe)
824 struct listen_ctx *lctx = synqe->lctx;
825 struct inpcb *inp = lctx->inp;
826 struct vi_info *vi = synqe->syn->m_pkthdr.rcvif->if_softc;
827 struct l2t_entry *e = &sc->l2t->l2tab[synqe->l2e_idx];
830 INP_WLOCK_ASSERT(inp);
831 ntids = inp->inp_vflag & INP_IPV6 ? 2 : 1;
833 TAILQ_REMOVE(&lctx->synq, synqe, link);
834 inp = release_lctx(sc, lctx);
837 remove_tid(sc, synqe->tid, ntids);
838 release_tid(sc, synqe->tid, &sc->sge.ctrlq[vi->pi->port_id]);
840 release_synqe(synqe); /* removed from synq list */
844 do_abort_req_synqe(struct sge_iq *iq, const struct rss_header *rss,
847 struct adapter *sc = iq->adapter;
848 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
849 unsigned int tid = GET_TID(cpl);
850 struct synq_entry *synqe = lookup_tid(sc, tid);
851 struct listen_ctx *lctx = synqe->lctx;
852 struct inpcb *inp = lctx->inp;
854 struct sge_wrq *ofld_txq;
856 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
859 KASSERT(opcode == CPL_ABORT_REQ_RSS,
860 ("%s: unexpected opcode 0x%x", __func__, opcode));
861 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
862 KASSERT(synqe->tid == tid, ("%s: toep tid mismatch", __func__));
864 CTR6(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x), lctx %p, status %d",
865 __func__, tid, synqe, synqe->flags, synqe->lctx, cpl->status);
867 if (negative_advice(cpl->status))
868 return (0); /* Ignore negative advice */
872 get_qids_from_mbuf(synqe->syn, &txqid, NULL);
873 ofld_txq = &sc->sge.ofld_txq[txqid];
876 * If we'd initiated an abort earlier the reply to it is responsible for
877 * cleaning up resources. Otherwise we tear everything down right here
878 * right now. We owe the T4 a CPL_ABORT_RPL no matter what.
880 if (synqe->flags & TPF_ABORT_SHUTDOWN) {
885 done_with_synqe(sc, synqe);
886 /* inp lock released by done_with_synqe */
888 send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
893 do_abort_rpl_synqe(struct sge_iq *iq, const struct rss_header *rss,
896 struct adapter *sc = iq->adapter;
897 const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
898 unsigned int tid = GET_TID(cpl);
899 struct synq_entry *synqe = lookup_tid(sc, tid);
900 struct listen_ctx *lctx = synqe->lctx;
901 struct inpcb *inp = lctx->inp;
903 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
906 KASSERT(opcode == CPL_ABORT_RPL_RSS,
907 ("%s: unexpected opcode 0x%x", __func__, opcode));
908 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
909 KASSERT(synqe->tid == tid, ("%s: toep tid mismatch", __func__));
911 CTR6(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x), lctx %p, status %d",
912 __func__, tid, synqe, synqe->flags, synqe->lctx, cpl->status);
915 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
916 ("%s: wasn't expecting abort reply for synqe %p (0x%x)",
917 __func__, synqe, synqe->flags));
919 done_with_synqe(sc, synqe);
920 /* inp lock released by done_with_synqe */
926 t4_offload_socket(struct toedev *tod, void *arg, struct socket *so)
928 struct adapter *sc = tod->tod_softc;
929 struct synq_entry *synqe = arg;
931 struct inpcb *inp = sotoinpcb(so);
933 struct cpl_pass_establish *cpl = mtod(synqe->syn, void *);
934 struct toepcb *toep = *(struct toepcb **)(cpl + 1);
936 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); /* prevents bad race with accept() */
937 INP_WLOCK_ASSERT(inp);
938 KASSERT(synqe->flags & TPF_SYNQE,
939 ("%s: %p not a synq_entry?", __func__, arg));
941 offload_socket(so, toep);
942 make_established(toep, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt);
943 toep->flags |= TPF_CPL_PENDING;
944 update_tid(sc, synqe->tid, toep);
945 synqe->flags |= TPF_SYNQE_EXPANDED;
949 save_qids_in_mbuf(struct mbuf *m, struct vi_info *vi)
951 uint32_t txqid, rxqid;
953 txqid = (arc4random() % vi->nofldtxq) + vi->first_ofld_txq;
954 rxqid = (arc4random() % vi->nofldrxq) + vi->first_ofld_rxq;
956 m->m_pkthdr.flowid = (txqid << 16) | (rxqid & 0xffff);
960 get_qids_from_mbuf(struct mbuf *m, int *txqid, int *rxqid)
964 *txqid = m->m_pkthdr.flowid >> 16;
966 *rxqid = m->m_pkthdr.flowid & 0xffff;
970 * Use the trailing space in the mbuf in which the PASS_ACCEPT_REQ arrived to
971 * store some state temporarily.
973 static struct synq_entry *
974 mbuf_to_synqe(struct mbuf *m)
976 int len = roundup2(sizeof (struct synq_entry), 8);
977 int tspace = M_TRAILINGSPACE(m);
978 struct synq_entry *synqe = NULL;
981 synqe = malloc(sizeof(*synqe), M_CXGBE, M_NOWAIT);
984 synqe->flags = TPF_SYNQE | TPF_SYNQE_NEEDFREE;
986 synqe = (void *)(m->m_data + m->m_len + tspace - len);
987 synqe->flags = TPF_SYNQE;
994 t4opt_to_tcpopt(const struct tcp_options *t4opt, struct tcpopt *to)
996 bzero(to, sizeof(*to));
999 to->to_flags |= TOF_MSS;
1000 to->to_mss = be16toh(t4opt->mss);
1004 to->to_flags |= TOF_SCALE;
1005 to->to_wscale = t4opt->wsf;
1009 to->to_flags |= TOF_TS;
1012 to->to_flags |= TOF_SACKPERM;
1016 * Options2 for passive open.
1019 calc_opt2p(struct adapter *sc, struct port_info *pi, int rxqid,
1020 const struct tcp_options *tcpopt, struct tcphdr *th, int ulp_mode)
1022 struct sge_ofld_rxq *ofld_rxq = &sc->sge.ofld_rxq[rxqid];
1025 opt2 = V_TX_QUEUE(sc->params.tp.tx_modq[pi->tx_chan]) |
1026 F_RSS_QUEUE_VALID | V_RSS_QUEUE(ofld_rxq->iq.abs_id);
1028 if (V_tcp_do_rfc1323) {
1030 opt2 |= F_TSTAMPS_EN;
1033 if (tcpopt->wsf <= 14)
1034 opt2 |= F_WND_SCALE_EN;
1037 if (V_tcp_do_ecn && th->th_flags & (TH_ECE | TH_CWR))
1038 opt2 |= F_CCTRL_ECN;
1040 /* RX_COALESCE is always a valid value (0 or M_RX_COALESCE). */
1042 opt2 |= F_RX_COALESCE_VALID;
1044 opt2 |= F_T5_OPT_2_VALID;
1045 opt2 |= F_CONG_CNTRL_VALID; /* OPT_2_ISS really, for T5 */
1047 if (sc->tt.rx_coalesce)
1048 opt2 |= V_RX_COALESCE(M_RX_COALESCE);
1050 #ifdef USE_DDP_RX_FLOW_CONTROL
1051 if (ulp_mode == ULP_MODE_TCPDDP)
1052 opt2 |= F_RX_FC_VALID | F_RX_FC_DDP;
1055 return htobe32(opt2);
1059 pass_accept_req_to_protohdrs(struct adapter *sc, const struct mbuf *m,
1060 struct in_conninfo *inc, struct tcphdr *th)
1062 const struct cpl_pass_accept_req *cpl = mtod(m, const void *);
1063 const struct ether_header *eh;
1064 unsigned int hlen = be32toh(cpl->hdr_len);
1066 const struct tcphdr *tcp;
1068 eh = (const void *)(cpl + 1);
1069 if (chip_id(sc) >= CHELSIO_T6) {
1070 l3hdr = ((uintptr_t)eh + G_T6_ETH_HDR_LEN(hlen));
1071 tcp = (const void *)(l3hdr + G_T6_IP_HDR_LEN(hlen));
1073 l3hdr = ((uintptr_t)eh + G_ETH_HDR_LEN(hlen));
1074 tcp = (const void *)(l3hdr + G_IP_HDR_LEN(hlen));
1078 bzero(inc, sizeof(*inc));
1079 inc->inc_fport = tcp->th_sport;
1080 inc->inc_lport = tcp->th_dport;
1081 if (((struct ip *)l3hdr)->ip_v == IPVERSION) {
1082 const struct ip *ip = (const void *)l3hdr;
1084 inc->inc_faddr = ip->ip_src;
1085 inc->inc_laddr = ip->ip_dst;
1087 const struct ip6_hdr *ip6 = (const void *)l3hdr;
1089 inc->inc_flags |= INC_ISIPV6;
1090 inc->inc6_faddr = ip6->ip6_src;
1091 inc->inc6_laddr = ip6->ip6_dst;
1096 bcopy(tcp, th, sizeof(*th));
1097 tcp_fields_to_host(th); /* just like tcp_input */
1101 static struct l2t_entry *
1102 get_l2te_for_nexthop(struct port_info *pi, struct ifnet *ifp,
1103 struct in_conninfo *inc)
1105 struct l2t_entry *e;
1106 struct sockaddr_in6 sin6;
1107 struct sockaddr *dst = (void *)&sin6;
1109 if (inc->inc_flags & INC_ISIPV6) {
1110 struct nhop6_basic nh6;
1112 bzero(dst, sizeof(struct sockaddr_in6));
1113 dst->sa_len = sizeof(struct sockaddr_in6);
1114 dst->sa_family = AF_INET6;
1116 if (IN6_IS_ADDR_LINKLOCAL(&inc->inc6_laddr)) {
1117 /* no need for route lookup */
1118 e = t4_l2t_get(pi, ifp, dst);
1122 if (fib6_lookup_nh_basic(RT_DEFAULT_FIB, &inc->inc6_faddr,
1123 0, 0, 0, &nh6) != 0)
1125 if (nh6.nh_ifp != ifp)
1127 ((struct sockaddr_in6 *)dst)->sin6_addr = nh6.nh_addr;
1129 struct nhop4_basic nh4;
1131 dst->sa_len = sizeof(struct sockaddr_in);
1132 dst->sa_family = AF_INET;
1134 if (fib4_lookup_nh_basic(RT_DEFAULT_FIB, inc->inc_faddr, 0, 0,
1137 if (nh4.nh_ifp != ifp)
1139 ((struct sockaddr_in *)dst)->sin_addr = nh4.nh_addr;
1142 e = t4_l2t_get(pi, ifp, dst);
1146 #define REJECT_PASS_ACCEPT() do { \
1147 reject_reason = __LINE__; \
1152 * The context associated with a tid entry via insert_tid could be a synq_entry
1153 * or a toepcb. The only way CPL handlers can tell is via a bit in these flags.
1155 CTASSERT(offsetof(struct toepcb, flags) == offsetof(struct synq_entry, flags));
1158 * Incoming SYN on a listening socket.
1160 * XXX: Every use of ifp in this routine has a bad race with up/down, toe/-toe,
1164 do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
1167 struct adapter *sc = iq->adapter;
1169 const struct cpl_pass_accept_req *cpl = mtod(m, const void *);
1170 struct cpl_pass_accept_rpl *rpl;
1172 unsigned int stid = G_PASS_OPEN_TID(be32toh(cpl->tos_stid));
1173 unsigned int tid = GET_TID(cpl);
1174 struct listen_ctx *lctx = lookup_stid(sc, stid);
1177 struct in_conninfo inc;
1180 struct port_info *pi;
1182 struct ifnet *hw_ifp, *ifp;
1183 struct l2t_entry *e = NULL;
1184 int rscale, mtu_idx, rx_credits, rxqid, ulp_mode;
1185 struct synq_entry *synqe = NULL;
1186 int reject_reason, v, ntids;
1189 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1192 KASSERT(opcode == CPL_PASS_ACCEPT_REQ,
1193 ("%s: unexpected opcode 0x%x", __func__, opcode));
1194 KASSERT(lctx->stid == stid, ("%s: lctx stid mismatch", __func__));
1196 CTR4(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p", __func__, stid, tid,
1199 pass_accept_req_to_protohdrs(sc, m, &inc, &th);
1200 t4opt_to_tcpopt(&cpl->tcpopt, &to);
1202 pi = sc->port[G_SYN_INTF(be16toh(cpl->l2info))];
1204 CURVNET_SET(lctx->vnet);
1207 * Use the MAC index to lookup the associated VI. If this SYN
1208 * didn't match a perfect MAC filter, punt.
1210 if (!(be16toh(cpl->l2info) & F_SYN_XACT_MATCH)) {
1213 REJECT_PASS_ACCEPT();
1215 for_each_vi(pi, v, vi) {
1216 if (vi->xact_addr_filt == G_SYN_MAC_IDX(be16toh(cpl->l2info)))
1221 REJECT_PASS_ACCEPT();
1224 hw_ifp = vi->ifp; /* the (v)cxgbeX ifnet */
1225 m->m_pkthdr.rcvif = hw_ifp;
1226 tod = TOEDEV(hw_ifp);
1229 * Figure out if there is a pseudo interface (vlan, lagg, etc.)
1230 * involved. Don't offload if the SYN had a VLAN tag and the vid
1231 * doesn't match anything on this interface.
1233 * XXX: lagg support, lagg + vlan support.
1235 vid = EVL_VLANOFTAG(be16toh(cpl->vlan));
1237 ifp = VLAN_DEVAT(hw_ifp, vid);
1239 REJECT_PASS_ACCEPT();
1244 * Don't offload if the peer requested a TCP option that's not known to
1247 if (cpl->tcpopt.unknown)
1248 REJECT_PASS_ACCEPT();
1250 if (inc.inc_flags & INC_ISIPV6) {
1252 /* Don't offload if the ifcap isn't enabled */
1253 if ((ifp->if_capenable & IFCAP_TOE6) == 0)
1254 REJECT_PASS_ACCEPT();
1257 * SYN must be directed to an IP6 address on this ifnet. This
1258 * is more restrictive than in6_localip.
1260 if (!in6_ifhasaddr(ifp, &inc.inc6_laddr))
1261 REJECT_PASS_ACCEPT();
1266 /* Don't offload if the ifcap isn't enabled */
1267 if ((ifp->if_capenable & IFCAP_TOE4) == 0)
1268 REJECT_PASS_ACCEPT();
1271 * SYN must be directed to an IP address on this ifnet. This
1272 * is more restrictive than in_localip.
1274 if (!in_ifhasaddr(ifp, inc.inc_laddr))
1275 REJECT_PASS_ACCEPT();
1281 * Don't offload if the ifnet that the SYN came in on is not in the same
1282 * vnet as the listening socket.
1284 if (lctx->vnet != ifp->if_vnet)
1285 REJECT_PASS_ACCEPT();
1287 e = get_l2te_for_nexthop(pi, ifp, &inc);
1289 REJECT_PASS_ACCEPT();
1291 synqe = mbuf_to_synqe(m);
1293 REJECT_PASS_ACCEPT();
1295 wr = alloc_wrqe(is_t4(sc) ? sizeof(struct cpl_pass_accept_rpl) :
1296 sizeof(struct cpl_t5_pass_accept_rpl), &sc->sge.ctrlq[pi->port_id]);
1298 REJECT_PASS_ACCEPT();
1301 INP_INFO_RLOCK(&V_tcbinfo); /* for 4-tuple check */
1303 /* Don't offload if the 4-tuple is already in use */
1304 if (toe_4tuple_check(&inc, &th, ifp) != 0) {
1305 INP_INFO_RUNLOCK(&V_tcbinfo);
1307 REJECT_PASS_ACCEPT();
1309 INP_INFO_RUNLOCK(&V_tcbinfo);
1311 inp = lctx->inp; /* listening socket, not owned by TOE */
1314 /* Don't offload if the listening socket has closed */
1315 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
1317 * The listening socket has closed. The reply from the TOE to
1318 * our CPL_CLOSE_LISTSRV_REQ will ultimately release all
1319 * resources tied to this listen context.
1323 REJECT_PASS_ACCEPT();
1325 so = inp->inp_socket;
1327 mtu_idx = find_best_mtu_idx(sc, &inc, be16toh(cpl->tcpopt.mss));
1328 rscale = cpl->tcpopt.wsf && V_tcp_do_rfc1323 ? select_rcv_wscale() : 0;
1329 SOCKBUF_LOCK(&so->so_rcv);
1330 /* opt0 rcv_bufsiz initially, assumes its normal meaning later */
1331 rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ);
1332 SOCKBUF_UNLOCK(&so->so_rcv);
1334 save_qids_in_mbuf(m, vi);
1335 get_qids_from_mbuf(m, NULL, &rxqid);
1338 INIT_TP_WR_MIT_CPL(rpl, CPL_PASS_ACCEPT_RPL, tid);
1340 struct cpl_t5_pass_accept_rpl *rpl5 = (void *)rpl;
1342 INIT_TP_WR_MIT_CPL(rpl5, CPL_PASS_ACCEPT_RPL, tid);
1344 if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0) {
1345 ulp_mode = ULP_MODE_TCPDDP;
1346 synqe->flags |= TPF_SYNQE_TCPDDP;
1348 ulp_mode = ULP_MODE_NONE;
1349 rpl->opt0 = calc_opt0(so, vi, e, mtu_idx, rscale, rx_credits, ulp_mode);
1350 rpl->opt2 = calc_opt2p(sc, pi, rxqid, &cpl->tcpopt, &th, ulp_mode);
1356 refcount_init(&synqe->refcnt, 1); /* 1 means extra hold */
1357 synqe->l2e_idx = e->idx;
1358 synqe->rcv_bufsize = rx_credits;
1359 atomic_store_rel_ptr(&synqe->wr, (uintptr_t)wr);
1361 insert_tid(sc, tid, synqe, ntids);
1362 TAILQ_INSERT_TAIL(&lctx->synq, synqe, link);
1363 hold_synqe(synqe); /* hold for the duration it's in the synq */
1364 hold_lctx(lctx); /* A synqe on the list has a ref on its lctx */
1367 * If all goes well t4_syncache_respond will get called during
1368 * syncache_add. Note that syncache_add releases the pcb lock.
1370 toe_syncache_add(&inc, &to, &th, inp, tod, synqe);
1371 INP_UNLOCK_ASSERT(inp); /* ok to assert, we have a ref on the inp */
1374 * If we replied during syncache_add (synqe->wr has been consumed),
1375 * good. Otherwise, set it to 0 so that further syncache_respond
1376 * attempts by the kernel will be ignored.
1378 if (atomic_cmpset_ptr(&synqe->wr, (uintptr_t)wr, 0)) {
1381 * syncache may or may not have a hold on the synqe, which may
1382 * or may not be stashed in the original SYN mbuf passed to us.
1383 * Just copy it over instead of dealing with all possibilities.
1385 m = m_dup(synqe->syn, M_NOWAIT);
1387 m->m_pkthdr.rcvif = hw_ifp;
1389 remove_tid(sc, synqe->tid, ntids);
1392 /* Yank the synqe out of the lctx synq. */
1394 TAILQ_REMOVE(&lctx->synq, synqe, link);
1395 release_synqe(synqe); /* removed from synq list */
1396 inp = release_lctx(sc, lctx);
1400 release_synqe(synqe); /* extra hold */
1401 REJECT_PASS_ACCEPT();
1404 CTR5(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p, synqe %p, SYNACK",
1405 __func__, stid, tid, lctx, synqe);
1408 synqe->flags |= TPF_SYNQE_HAS_L2TE;
1409 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
1411 * Listening socket closed but tod_listen_stop did not abort
1412 * this tid because there was no L2T entry for the tid at that
1413 * time. Abort it now. The reply to the abort will clean up.
1416 "%s: stid %u, tid %u, lctx %p, synqe %p (0x%x), ABORT",
1417 __func__, stid, tid, lctx, synqe, synqe->flags);
1418 if (!(synqe->flags & TPF_SYNQE_EXPANDED))
1419 send_reset_synqe(tod, synqe);
1423 release_synqe(synqe); /* extra hold */
1429 release_synqe(synqe); /* extra hold */
1433 CTR4(KTR_CXGBE, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid,
1438 release_tid(sc, tid, lctx->ctrlq);
1440 if (__predict_true(m != NULL)) {
1441 m_adj(m, sizeof(*cpl));
1442 m->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID |
1443 CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1444 m->m_pkthdr.csum_data = 0xffff;
1445 hw_ifp->if_input(hw_ifp, m);
1448 return (reject_reason);
1452 synqe_to_protohdrs(struct adapter *sc, struct synq_entry *synqe,
1453 const struct cpl_pass_establish *cpl, struct in_conninfo *inc,
1454 struct tcphdr *th, struct tcpopt *to)
1456 uint16_t tcp_opt = be16toh(cpl->tcp_opt);
1458 /* start off with the original SYN */
1459 pass_accept_req_to_protohdrs(sc, synqe->syn, inc, th);
1461 /* modify parts to make it look like the ACK to our SYN|ACK */
1462 th->th_flags = TH_ACK;
1463 th->th_ack = synqe->iss + 1;
1464 th->th_seq = be32toh(cpl->rcv_isn);
1465 bzero(to, sizeof(*to));
1466 if (G_TCPOPT_TSTAMP(tcp_opt)) {
1467 to->to_flags |= TOF_TS;
1468 to->to_tsecr = synqe->ts;
1473 do_pass_establish(struct sge_iq *iq, const struct rss_header *rss,
1476 struct adapter *sc = iq->adapter;
1479 const struct cpl_pass_establish *cpl = (const void *)(rss + 1);
1480 #if defined(KTR) || defined(INVARIANTS)
1481 unsigned int stid = G_PASS_OPEN_TID(be32toh(cpl->tos_stid));
1483 unsigned int tid = GET_TID(cpl);
1484 struct synq_entry *synqe = lookup_tid(sc, tid);
1485 struct listen_ctx *lctx = synqe->lctx;
1486 struct inpcb *inp = lctx->inp, *new_inp;
1490 struct in_conninfo inc;
1491 struct toepcb *toep;
1494 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
1497 KASSERT(opcode == CPL_PASS_ESTABLISH,
1498 ("%s: unexpected opcode 0x%x", __func__, opcode));
1499 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1500 KASSERT(lctx->stid == stid, ("%s: lctx stid mismatch", __func__));
1501 KASSERT(synqe->flags & TPF_SYNQE,
1502 ("%s: tid %u (ctx %p) not a synqe", __func__, tid, synqe));
1504 CURVNET_SET(lctx->vnet);
1505 INP_INFO_RLOCK(&V_tcbinfo); /* for syncache_expand */
1509 "%s: stid %u, tid %u, synqe %p (0x%x), inp_flags 0x%x",
1510 __func__, stid, tid, synqe, synqe->flags, inp->inp_flags);
1512 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
1514 if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
1515 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
1516 ("%s: listen socket closed but tid %u not aborted.",
1521 INP_INFO_RUNLOCK(&V_tcbinfo);
1526 ifp = synqe->syn->m_pkthdr.rcvif;
1528 KASSERT(vi->pi->adapter == sc,
1529 ("%s: vi %p, sc %p mismatch", __func__, vi, sc));
1531 get_qids_from_mbuf(synqe->syn, &txqid, &rxqid);
1532 KASSERT(rxqid == iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0],
1533 ("%s: CPL arrived on unexpected rxq. %d %d", __func__, rxqid,
1534 (int)(iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0])));
1536 toep = alloc_toepcb(vi, txqid, rxqid, M_NOWAIT);
1540 * The reply to this abort will perform final cleanup. There is
1541 * no need to check for HAS_L2TE here. We can be here only if
1542 * we responded to the PASS_ACCEPT_REQ, and our response had the
1545 send_reset_synqe(TOEDEV(ifp), synqe);
1547 INP_INFO_RUNLOCK(&V_tcbinfo);
1552 toep->l2te = &sc->l2t->l2tab[synqe->l2e_idx];
1553 if (synqe->flags & TPF_SYNQE_TCPDDP)
1554 set_tcpddp_ulp_mode(toep);
1556 toep->ulp_mode = ULP_MODE_NONE;
1557 /* opt0 rcv_bufsiz initially, assumes its normal meaning later */
1558 toep->rx_credits = synqe->rcv_bufsize;
1560 so = inp->inp_socket;
1561 KASSERT(so != NULL, ("%s: socket is NULL", __func__));
1563 /* Come up with something that syncache_expand should be ok with. */
1564 synqe_to_protohdrs(sc, synqe, cpl, &inc, &th, &to);
1567 * No more need for anything in the mbuf that carried the
1568 * CPL_PASS_ACCEPT_REQ. Drop the CPL_PASS_ESTABLISH and toep pointer
1569 * there. XXX: bad form but I don't want to increase the size of synqe.
1572 KASSERT(sizeof(*cpl) + sizeof(toep) <= m->m_len,
1573 ("%s: no room in mbuf %p (m_len %d)", __func__, m, m->m_len));
1574 bcopy(cpl, mtod(m, void *), sizeof(*cpl));
1575 *(struct toepcb **)(mtod(m, struct cpl_pass_establish *) + 1) = toep;
1577 if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) {
1582 /* New connection inpcb is already locked by syncache_expand(). */
1583 new_inp = sotoinpcb(so);
1584 INP_WLOCK_ASSERT(new_inp);
1585 MPASS(so->so_vnet == lctx->vnet);
1586 toep->vnet = lctx->vnet;
1587 if (inc.inc_flags & INC_ISIPV6)
1588 toep->ce = hold_lip(sc->tom_softc, &inc.inc6_laddr, lctx->ce);
1591 * This is for the unlikely case where the syncache entry that we added
1592 * has been evicted from the syncache, but the syncache_expand above
1593 * works because of syncookies.
1595 * XXX: we've held the tcbinfo lock throughout so there's no risk of
1596 * anyone accept'ing a connection before we've installed our hooks, but
1597 * this somewhat defeats the purpose of having a tod_offload_socket :-(
1599 if (__predict_false(!(synqe->flags & TPF_SYNQE_EXPANDED))) {
1600 tcp_timer_activate(intotcpcb(new_inp), TT_KEEP, 0);
1601 t4_offload_socket(TOEDEV(ifp), synqe, so);
1604 INP_WUNLOCK(new_inp);
1606 /* Done with the synqe */
1607 TAILQ_REMOVE(&lctx->synq, synqe, link);
1608 inp = release_lctx(sc, lctx);
1611 INP_INFO_RUNLOCK(&V_tcbinfo);
1613 release_synqe(synqe);
1619 t4_init_listen_cpl_handlers(void)
1622 t4_register_cpl_handler(CPL_PASS_OPEN_RPL, do_pass_open_rpl);
1623 t4_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_close_server_rpl);
1624 t4_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_pass_accept_req);
1625 t4_register_cpl_handler(CPL_PASS_ESTABLISH, do_pass_establish);
1629 t4_uninit_listen_cpl_handlers(void)
1632 t4_register_cpl_handler(CPL_PASS_OPEN_RPL, NULL);
1633 t4_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, NULL);
1634 t4_register_cpl_handler(CPL_PASS_ACCEPT_REQ, NULL);
1635 t4_register_cpl_handler(CPL_PASS_ESTABLISH, NULL);