sys/dev/cxgbe/tom/t4_connect.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012 Chelsio Communications, Inc.
   5  * All rights reserved.
   6  * Written by: Navdeep Parhar <np@FreeBSD.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_inet.h"
  34 #include "opt_inet6.h"
  35
  36 #ifdef TCP_OFFLOAD
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/ktr.h>
  41 #include <sys/module.h>
  42 #include <sys/protosw.h>
  43 #include <sys/domain.h>
  44 #include <sys/socket.h>
  45 #include <sys/socketvar.h>
  46 #include <sys/sysctl.h>
  47 #include <net/ethernet.h>
  48 #include <net/if.h>
  49 #include <net/if_types.h>
  50 #include <net/if_vlan_var.h>
  51 #include <net/route.h>
  52 #include <netinet/in.h>
  53 #include <netinet/in_pcb.h>
  54 #include <netinet/ip.h>
  55 #define TCPSTATES
  56 #include <netinet/tcp_fsm.h>
  57 #include <netinet/tcp_var.h>
  58 #include <netinet/toecore.h>
  59 #include <netinet/cc/cc.h>
  60
  61 #include "common/common.h"
  62 #include "common/t4_msg.h"
  63 #include "common/t4_regs.h"
  64 #include "common/t4_regs_values.h"
  65 #include "t4_clip.h"
  66 #include "tom/t4_tom_l2t.h"
  67 #include "tom/t4_tom.h"
  68
  69 /*
  70  * Active open succeeded.
  71  */
  72 static int
  73 do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
  74     struct mbuf *m)
  75 {
  76         struct adapter *sc = iq->adapter;
  77         const struct cpl_act_establish *cpl = (const void *)(rss + 1);
  78         u_int tid = GET_TID(cpl);
  79         u_int atid = G_TID_TID(ntohl(cpl->tos_atid));
  80         struct toepcb *toep = lookup_atid(sc, atid);
  81         struct inpcb *inp = toep->inp;
  82
  83         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
  84         KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
  85
  86         CTR3(KTR_CXGBE, "%s: atid %u, tid %u", __func__, atid, tid);
  87         free_atid(sc, atid);
  88
  89         CURVNET_SET(toep->vnet);
  90         INP_WLOCK(inp);
  91         toep->tid = tid;
  92         insert_tid(sc, tid, toep, inp->inp_vflag & INP_IPV6 ? 2 : 1);
  93         if (inp->inp_flags & INP_DROPPED) {
  94
  95                 /* socket closed by the kernel before hw told us it connected */
  96
  97                 send_flowc_wr(toep, NULL);
  98                 send_reset(sc, toep, be32toh(cpl->snd_isn));
  99                 goto done;
 100         }
 101
 102         make_established(toep, be32toh(cpl->snd_isn) - 1,
 103             be32toh(cpl->rcv_isn) - 1, cpl->tcp_opt);
 104
 105         if (ulp_mode(toep) == ULP_MODE_TLS)
 106                 tls_establish(toep);
 107
 108 done:
 109         INP_WUNLOCK(inp);
 110         CURVNET_RESTORE();
 111         return (0);
 112 }
 113
 114 void
 115 act_open_failure_cleanup(struct adapter *sc, u_int atid, u_int status)
 116 {
 117         struct toepcb *toep = lookup_atid(sc, atid);
 118         struct inpcb *inp = toep->inp;
 119         struct toedev *tod = &toep->td->tod;
 120         struct epoch_tracker et;
 121
 122         free_atid(sc, atid);
 123         toep->tid = -1;
 124
 125         CURVNET_SET(toep->vnet);
 126         if (status != EAGAIN)
 127                 NET_EPOCH_ENTER(et);
 128         INP_WLOCK(inp);
 129         toe_connect_failed(tod, inp, status);
 130         final_cpl_received(toep);       /* unlocks inp */
 131         if (status != EAGAIN)
 132                 NET_EPOCH_EXIT(et);
 133         CURVNET_RESTORE();
 134 }
 135
 136 /*
 137  * Active open failed.
 138  */
 139 static int
 140 do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
 141     struct mbuf *m)
 142 {
 143         struct adapter *sc = iq->adapter;
 144         const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
 145         u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
 146         u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
 147         struct toepcb *toep = lookup_atid(sc, atid);
 148         int rc;
 149
 150         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 151         KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
 152
 153         CTR3(KTR_CXGBE, "%s: atid %u, status %u ", __func__, atid, status);
 154
 155         /* Ignore negative advice */
 156         if (negative_advice(status))
 157                 return (0);
 158
 159         if (status && act_open_has_tid(status))
 160                 release_tid(sc, GET_TID(cpl), toep->ctrlq);
 161
 162         rc = act_open_rpl_status_to_errno(status);
 163         act_open_failure_cleanup(sc, atid, rc);
 164
 165         return (0);
 166 }
 167
 168 void
 169 t4_init_connect_cpl_handlers(void)
 170 {
 171
 172         t4_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
 173         t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl,
 174             CPL_COOKIE_TOM);
 175 }
 176
 177 void
 178 t4_uninit_connect_cpl_handlers(void)
 179 {
 180
 181         t4_register_cpl_handler(CPL_ACT_ESTABLISH, NULL);
 182         t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL, CPL_COOKIE_TOM);
 183 }
 184
 185 #define DONT_OFFLOAD_ACTIVE_OPEN(x)     do { \
 186         reason = __LINE__; \
 187         rc = (x); \
 188         goto failed; \
 189 } while (0)
 190
 191 static inline int
 192 act_open_cpl_size(struct adapter *sc, int isipv6)
 193 {
 194         int idx;
 195         static const int sz_table[3][2] = {
 196                 {
 197                         sizeof (struct cpl_act_open_req),
 198                         sizeof (struct cpl_act_open_req6)
 199                 },
 200                 {
 201                         sizeof (struct cpl_t5_act_open_req),
 202                         sizeof (struct cpl_t5_act_open_req6)
 203                 },
 204                 {
 205                         sizeof (struct cpl_t6_act_open_req),
 206                         sizeof (struct cpl_t6_act_open_req6)
 207                 },
 208         };
 209
 210         MPASS(chip_id(sc) >= CHELSIO_T4);
 211         idx = min(chip_id(sc) - CHELSIO_T4, 2);
 212
 213         return (sz_table[idx][!!isipv6]);
 214 }
 215
 216 /*
 217  * active open (soconnect).
 218  *
 219  * State of affairs on entry:
 220  * soisconnecting (so_state |= SS_ISCONNECTING)
 221  * tcbinfo not locked (This has changed - used to be WLOCKed)
 222  * inp WLOCKed
 223  * tp->t_state = TCPS_SYN_SENT
 224  * rtalloc1, RT_UNLOCK on rt.
 225  */
 226 int
 227 t4_connect(struct toedev *tod, struct socket *so, struct rtentry *rt,
 228     struct sockaddr *nam)
 229 {
 230         struct adapter *sc = tod->tod_softc;
 231         struct toepcb *toep = NULL;
 232         struct wrqe *wr = NULL;
 233         struct ifnet *rt_ifp = rt->rt_ifp;
 234         struct vi_info *vi;
 235         int qid_atid, rc, isipv6;
 236         struct inpcb *inp = sotoinpcb(so);
 237         struct tcpcb *tp = intotcpcb(inp);
 238         int reason;
 239         struct offload_settings settings;
 240         struct epoch_tracker et;
 241         uint16_t vid = 0xfff, pcp = 0;
 242
 243         INP_WLOCK_ASSERT(inp);
 244         KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6,
 245             ("%s: dest addr %p has family %u", __func__, nam, nam->sa_family));
 246
 247         if (rt_ifp->if_type == IFT_ETHER)
 248                 vi = rt_ifp->if_softc;
 249         else if (rt_ifp->if_type == IFT_L2VLAN) {
 250                 struct ifnet *ifp = VLAN_TRUNKDEV(rt_ifp);
 251
 252                 vi = ifp->if_softc;
 253                 VLAN_TAG(rt_ifp, &vid);
 254                 VLAN_PCP(rt_ifp, &pcp);
 255         } else if (rt_ifp->if_type == IFT_IEEE8023ADLAG)
 256                 DONT_OFFLOAD_ACTIVE_OPEN(ENOSYS); /* XXX: implement lagg+TOE */
 257         else
 258                 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 259         if (sc->flags & KERN_TLS_OK)
 260                 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 261
 262         rw_rlock(&sc->policy_lock);
 263         settings = *lookup_offload_policy(sc, OPEN_TYPE_ACTIVE, NULL,
 264             EVL_MAKETAG(vid, pcp, 0), inp);
 265         rw_runlock(&sc->policy_lock);
 266         if (!settings.offload)
 267                 DONT_OFFLOAD_ACTIVE_OPEN(EPERM);
 268
 269         toep = alloc_toepcb(vi, M_NOWAIT);
 270         if (toep == NULL)
 271                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 272
 273         toep->tid = alloc_atid(sc, toep);
 274         if (toep->tid < 0)
 275                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 276
 277         toep->l2te = t4_l2t_get(vi->pi, rt_ifp,
 278             rt->rt_flags & RTF_GATEWAY ? rt->rt_gateway : nam);
 279         if (toep->l2te == NULL)
 280                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 281
 282         toep->vnet = so->so_vnet;
 283         init_conn_params(vi, &settings, &inp->inp_inc, so, NULL,
 284             toep->l2te->idx, &toep->params);
 285         init_toepcb(vi, toep);
 286
 287         isipv6 = nam->sa_family == AF_INET6;
 288         wr = alloc_wrqe(act_open_cpl_size(sc, isipv6), toep->ctrlq);
 289         if (wr == NULL)
 290                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 291
 292         qid_atid = V_TID_QID(toep->ofld_rxq->iq.abs_id) | V_TID_TID(toep->tid) |
 293             V_TID_COOKIE(CPL_COOKIE_TOM);
 294
 295         if (isipv6) {
 296                 struct cpl_act_open_req6 *cpl = wrtod(wr);
 297                 struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
 298                 struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
 299
 300                 if ((inp->inp_vflag & INP_IPV6) == 0)
 301                         DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 302
 303                 toep->ce = t4_hold_lip(sc, &inp->in6p_laddr, NULL);
 304                 if (toep->ce == NULL)
 305                         DONT_OFFLOAD_ACTIVE_OPEN(ENOENT);
 306
 307                 switch (chip_id(sc)) {
 308                 case CHELSIO_T4:
 309                         INIT_TP_WR(cpl, 0);
 310                         cpl->params = select_ntuple(vi, toep->l2te);
 311                         break;
 312                 case CHELSIO_T5:
 313                         INIT_TP_WR(cpl5, 0);
 314                         cpl5->iss = htobe32(tp->iss);
 315                         cpl5->params = select_ntuple(vi, toep->l2te);
 316                         break;
 317                 case CHELSIO_T6:
 318                 default:
 319                         INIT_TP_WR(cpl6, 0);
 320                         cpl6->iss = htobe32(tp->iss);
 321                         cpl6->params = select_ntuple(vi, toep->l2te);
 322                         break;
 323                 }
 324                 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
 325                     qid_atid));
 326                 cpl->local_port = inp->inp_lport;
 327                 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
 328                 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
 329                 cpl->peer_port = inp->inp_fport;
 330                 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
 331                 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
 332                 cpl->opt0 = calc_options0(vi, &toep->params);
 333                 cpl->opt2 = calc_options2(vi, &toep->params);
 334
 335                 CTR6(KTR_CXGBE,
 336                     "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
 337                     __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
 338                     be32toh(cpl->opt2));
 339         } else {
 340                 struct cpl_act_open_req *cpl = wrtod(wr);
 341                 struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
 342                 struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
 343
 344                 switch (chip_id(sc)) {
 345                 case CHELSIO_T4:
 346                         INIT_TP_WR(cpl, 0);
 347                         cpl->params = select_ntuple(vi, toep->l2te);
 348                         break;
 349                 case CHELSIO_T5:
 350                         INIT_TP_WR(cpl5, 0);
 351                         cpl5->iss = htobe32(tp->iss);
 352                         cpl5->params = select_ntuple(vi, toep->l2te);
 353                         break;
 354                 case CHELSIO_T6:
 355                 default:
 356                         INIT_TP_WR(cpl6, 0);
 357                         cpl6->iss = htobe32(tp->iss);
 358                         cpl6->params = select_ntuple(vi, toep->l2te);
 359                         break;
 360                 }
 361                 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
 362                     qid_atid));
 363                 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port,
 364                     &cpl->peer_ip, &cpl->peer_port);
 365                 cpl->opt0 = calc_options0(vi, &toep->params);
 366                 cpl->opt2 = calc_options2(vi, &toep->params);
 367
 368                 CTR6(KTR_CXGBE,
 369                     "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
 370                     __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
 371                     be32toh(cpl->opt2));
 372         }
 373
 374         offload_socket(so, toep);
 375         NET_EPOCH_ENTER(et);
 376         rc = t4_l2t_send(sc, wr, toep->l2te);
 377         NET_EPOCH_EXIT(et);
 378         if (rc == 0) {
 379                 toep->flags |= TPF_CPL_PENDING;
 380                 return (0);
 381         }
 382
 383         undo_offload_socket(so);
 384         reason = __LINE__;
 385 failed:
 386         CTR3(KTR_CXGBE, "%s: not offloading (%d), rc %d", __func__, reason, rc);
 387
 388         if (wr)
 389                 free_wrqe(wr);
 390
 391         if (toep) {
 392                 if (toep->tid >= 0)
 393                         free_atid(sc, toep->tid);
 394                 if (toep->l2te)
 395                         t4_l2t_release(toep->l2te);
 396                 if (toep->ce)
 397                         t4_release_lip(sc, toep->ce);
 398                 free_toepcb(toep);
 399         }
 400
 401         return (rc);
 402 }
 403 #endif