sys/dev/cxgbe/tom/t4_connect.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012 Chelsio Communications, Inc.
   5  * All rights reserved.
   6  * Written by: Navdeep Parhar <np@FreeBSD.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_inet.h"
  34 #include "opt_inet6.h"
  35
  36 #ifdef TCP_OFFLOAD
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/ktr.h>
  41 #include <sys/module.h>
  42 #include <sys/protosw.h>
  43 #include <sys/domain.h>
  44 #include <sys/socket.h>
  45 #include <sys/socketvar.h>
  46 #include <sys/sysctl.h>
  47 #include <net/ethernet.h>
  48 #include <net/if.h>
  49 #include <net/if_types.h>
  50 #include <net/if_vlan_var.h>
  51 #include <net/route.h>
  52 #include <net/route/nhop.h>
  53 #include <netinet/in.h>
  54 #include <netinet/in_pcb.h>
  55 #include <netinet/ip.h>
  56 #define TCPSTATES
  57 #include <netinet/tcp_fsm.h>
  58 #include <netinet/tcp_var.h>
  59 #include <netinet/toecore.h>
  60 #include <netinet/cc/cc.h>
  61
  62 #include "common/common.h"
  63 #include "common/t4_msg.h"
  64 #include "common/t4_regs.h"
  65 #include "common/t4_regs_values.h"
  66 #include "t4_clip.h"
  67 #include "tom/t4_tom_l2t.h"
  68 #include "tom/t4_tom.h"
  69
  70 /*
  71  * Active open succeeded.
  72  */
  73 static int
  74 do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
  75     struct mbuf *m)
  76 {
  77         struct adapter *sc = iq->adapter;
  78         const struct cpl_act_establish *cpl = (const void *)(rss + 1);
  79         u_int tid = GET_TID(cpl);
  80         u_int atid = G_TID_TID(ntohl(cpl->tos_atid));
  81         struct toepcb *toep = lookup_atid(sc, atid);
  82         struct inpcb *inp = toep->inp;
  83
  84         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
  85         KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
  86
  87         CTR3(KTR_CXGBE, "%s: atid %u, tid %u", __func__, atid, tid);
  88         free_atid(sc, atid);
  89
  90         CURVNET_SET(toep->vnet);
  91         INP_WLOCK(inp);
  92         toep->tid = tid;
  93         insert_tid(sc, tid, toep, inp->inp_vflag & INP_IPV6 ? 2 : 1);
  94         if (inp->inp_flags & INP_DROPPED) {
  95
  96                 /* socket closed by the kernel before hw told us it connected */
  97
  98                 send_flowc_wr(toep, NULL);
  99                 send_reset(sc, toep, be32toh(cpl->snd_isn));
 100                 goto done;
 101         }
 102
 103         make_established(toep, be32toh(cpl->snd_isn) - 1,
 104             be32toh(cpl->rcv_isn) - 1, cpl->tcp_opt);
 105         inp->inp_flowtype = M_HASHTYPE_OPAQUE;
 106         inp->inp_flowid = tid;
 107
 108         if (ulp_mode(toep) == ULP_MODE_TLS)
 109                 tls_establish(toep);
 110
 111 done:
 112         INP_WUNLOCK(inp);
 113         CURVNET_RESTORE();
 114         return (0);
 115 }
 116
 117 void
 118 act_open_failure_cleanup(struct adapter *sc, u_int atid, u_int status)
 119 {
 120         struct toepcb *toep = lookup_atid(sc, atid);
 121         struct inpcb *inp = toep->inp;
 122         struct toedev *tod = &toep->td->tod;
 123         struct epoch_tracker et;
 124
 125         free_atid(sc, atid);
 126         toep->tid = -1;
 127
 128         CURVNET_SET(toep->vnet);
 129         if (status != EAGAIN)
 130                 NET_EPOCH_ENTER(et);
 131         INP_WLOCK(inp);
 132         toe_connect_failed(tod, inp, status);
 133         final_cpl_received(toep);       /* unlocks inp */
 134         if (status != EAGAIN)
 135                 NET_EPOCH_EXIT(et);
 136         CURVNET_RESTORE();
 137 }
 138
 139 /*
 140  * Active open failed.
 141  */
 142 static int
 143 do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
 144     struct mbuf *m)
 145 {
 146         struct adapter *sc = iq->adapter;
 147         const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
 148         u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
 149         u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
 150         struct toepcb *toep = lookup_atid(sc, atid);
 151         int rc;
 152
 153         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 154         KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
 155
 156         CTR3(KTR_CXGBE, "%s: atid %u, status %u ", __func__, atid, status);
 157
 158         /* Ignore negative advice */
 159         if (negative_advice(status))
 160                 return (0);
 161
 162         if (status && act_open_has_tid(status))
 163                 release_tid(sc, GET_TID(cpl), toep->ctrlq);
 164
 165         rc = act_open_rpl_status_to_errno(status);
 166         act_open_failure_cleanup(sc, atid, rc);
 167
 168         return (0);
 169 }
 170
 171 void
 172 t4_init_connect_cpl_handlers(void)
 173 {
 174
 175         t4_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
 176         t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl,
 177             CPL_COOKIE_TOM);
 178 }
 179
 180 void
 181 t4_uninit_connect_cpl_handlers(void)
 182 {
 183
 184         t4_register_cpl_handler(CPL_ACT_ESTABLISH, NULL);
 185         t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL, CPL_COOKIE_TOM);
 186 }
 187
 188 #define DONT_OFFLOAD_ACTIVE_OPEN(x)     do { \
 189         reason = __LINE__; \
 190         rc = (x); \
 191         goto failed; \
 192 } while (0)
 193
 194 static inline int
 195 act_open_cpl_size(struct adapter *sc, int isipv6)
 196 {
 197         int idx;
 198         static const int sz_table[3][2] = {
 199                 {
 200                         sizeof (struct cpl_act_open_req),
 201                         sizeof (struct cpl_act_open_req6)
 202                 },
 203                 {
 204                         sizeof (struct cpl_t5_act_open_req),
 205                         sizeof (struct cpl_t5_act_open_req6)
 206                 },
 207                 {
 208                         sizeof (struct cpl_t6_act_open_req),
 209                         sizeof (struct cpl_t6_act_open_req6)
 210                 },
 211         };
 212
 213         MPASS(chip_id(sc) >= CHELSIO_T4);
 214         idx = min(chip_id(sc) - CHELSIO_T4, 2);
 215
 216         return (sz_table[idx][!!isipv6]);
 217 }
 218
 219 /*
 220  * active open (soconnect).
 221  *
 222  * State of affairs on entry:
 223  * soisconnecting (so_state |= SS_ISCONNECTING)
 224  * tcbinfo not locked (This has changed - used to be WLOCKed)
 225  * inp WLOCKed
 226  * tp->t_state = TCPS_SYN_SENT
 227  * rtalloc1, RT_UNLOCK on rt.
 228  */
 229 int
 230 t4_connect(struct toedev *tod, struct socket *so, struct nhop_object *nh,
 231     struct sockaddr *nam)
 232 {
 233         struct adapter *sc = tod->tod_softc;
 234         struct toepcb *toep = NULL;
 235         struct wrqe *wr = NULL;
 236         struct ifnet *rt_ifp = nh->nh_ifp;
 237         struct vi_info *vi;
 238         int qid_atid, rc, isipv6;
 239         struct inpcb *inp = sotoinpcb(so);
 240         struct tcpcb *tp = intotcpcb(inp);
 241         int reason;
 242         struct offload_settings settings;
 243         struct epoch_tracker et;
 244         uint16_t vid = 0xfff, pcp = 0;
 245
 246         INP_WLOCK_ASSERT(inp);
 247         KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6,
 248             ("%s: dest addr %p has family %u", __func__, nam, nam->sa_family));
 249
 250         if (rt_ifp->if_type == IFT_ETHER)
 251                 vi = rt_ifp->if_softc;
 252         else if (rt_ifp->if_type == IFT_L2VLAN) {
 253                 struct ifnet *ifp = VLAN_TRUNKDEV(rt_ifp);
 254
 255                 vi = ifp->if_softc;
 256                 VLAN_TAG(rt_ifp, &vid);
 257                 VLAN_PCP(rt_ifp, &pcp);
 258         } else if (rt_ifp->if_type == IFT_IEEE8023ADLAG)
 259                 DONT_OFFLOAD_ACTIVE_OPEN(ENOSYS); /* XXX: implement lagg+TOE */
 260         else
 261                 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 262         if (sc->flags & KERN_TLS_OK)
 263                 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 264
 265         rw_rlock(&sc->policy_lock);
 266         settings = *lookup_offload_policy(sc, OPEN_TYPE_ACTIVE, NULL,
 267             EVL_MAKETAG(vid, pcp, 0), inp);
 268         rw_runlock(&sc->policy_lock);
 269         if (!settings.offload)
 270                 DONT_OFFLOAD_ACTIVE_OPEN(EPERM);
 271
 272         toep = alloc_toepcb(vi, M_NOWAIT);
 273         if (toep == NULL)
 274                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 275
 276         toep->tid = alloc_atid(sc, toep);
 277         if (toep->tid < 0)
 278                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 279
 280         toep->l2te = t4_l2t_get(vi->pi, rt_ifp,
 281             nh->nh_flags & NHF_GATEWAY ? &nh->gw_sa : nam);
 282         if (toep->l2te == NULL)
 283                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 284
 285         toep->vnet = so->so_vnet;
 286         init_conn_params(vi, &settings, &inp->inp_inc, so, NULL,
 287             toep->l2te->idx, &toep->params);
 288         init_toepcb(vi, toep);
 289
 290         isipv6 = nam->sa_family == AF_INET6;
 291         wr = alloc_wrqe(act_open_cpl_size(sc, isipv6), toep->ctrlq);
 292         if (wr == NULL)
 293                 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 294
 295         qid_atid = V_TID_QID(toep->ofld_rxq->iq.abs_id) | V_TID_TID(toep->tid) |
 296             V_TID_COOKIE(CPL_COOKIE_TOM);
 297
 298         if (isipv6) {
 299                 struct cpl_act_open_req6 *cpl = wrtod(wr);
 300                 struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
 301                 struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
 302
 303                 if ((inp->inp_vflag & INP_IPV6) == 0)
 304                         DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 305
 306                 toep->ce = t4_hold_lip(sc, &inp->in6p_laddr, NULL);
 307                 if (toep->ce == NULL)
 308                         DONT_OFFLOAD_ACTIVE_OPEN(ENOENT);
 309
 310                 switch (chip_id(sc)) {
 311                 case CHELSIO_T4:
 312                         INIT_TP_WR(cpl, 0);
 313                         cpl->params = select_ntuple(vi, toep->l2te);
 314                         break;
 315                 case CHELSIO_T5:
 316                         INIT_TP_WR(cpl5, 0);
 317                         cpl5->iss = htobe32(tp->iss);
 318                         cpl5->params = select_ntuple(vi, toep->l2te);
 319                         break;
 320                 case CHELSIO_T6:
 321                 default:
 322                         INIT_TP_WR(cpl6, 0);
 323                         cpl6->iss = htobe32(tp->iss);
 324                         cpl6->params = select_ntuple(vi, toep->l2te);
 325                         break;
 326                 }
 327                 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
 328                     qid_atid));
 329                 cpl->local_port = inp->inp_lport;
 330                 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
 331                 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
 332                 cpl->peer_port = inp->inp_fport;
 333                 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
 334                 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
 335                 cpl->opt0 = calc_options0(vi, &toep->params);
 336                 cpl->opt2 = calc_options2(vi, &toep->params);
 337
 338                 CTR6(KTR_CXGBE,
 339                     "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
 340                     __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
 341                     be32toh(cpl->opt2));
 342         } else {
 343                 struct cpl_act_open_req *cpl = wrtod(wr);
 344                 struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
 345                 struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
 346
 347                 switch (chip_id(sc)) {
 348                 case CHELSIO_T4:
 349                         INIT_TP_WR(cpl, 0);
 350                         cpl->params = select_ntuple(vi, toep->l2te);
 351                         break;
 352                 case CHELSIO_T5:
 353                         INIT_TP_WR(cpl5, 0);
 354                         cpl5->iss = htobe32(tp->iss);
 355                         cpl5->params = select_ntuple(vi, toep->l2te);
 356                         break;
 357                 case CHELSIO_T6:
 358                 default:
 359                         INIT_TP_WR(cpl6, 0);
 360                         cpl6->iss = htobe32(tp->iss);
 361                         cpl6->params = select_ntuple(vi, toep->l2te);
 362                         break;
 363                 }
 364                 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
 365                     qid_atid));
 366                 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port,
 367                     &cpl->peer_ip, &cpl->peer_port);
 368                 cpl->opt0 = calc_options0(vi, &toep->params);
 369                 cpl->opt2 = calc_options2(vi, &toep->params);
 370
 371                 CTR6(KTR_CXGBE,
 372                     "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
 373                     __func__, toep->tid, toep, inp, be64toh(cpl->opt0),
 374                     be32toh(cpl->opt2));
 375         }
 376
 377         offload_socket(so, toep);
 378         NET_EPOCH_ENTER(et);
 379         rc = t4_l2t_send(sc, wr, toep->l2te);
 380         NET_EPOCH_EXIT(et);
 381         if (rc == 0) {
 382                 toep->flags |= TPF_CPL_PENDING;
 383                 return (0);
 384         }
 385
 386         undo_offload_socket(so);
 387         reason = __LINE__;
 388 failed:
 389         CTR3(KTR_CXGBE, "%s: not offloading (%d), rc %d", __func__, reason, rc);
 390
 391         if (wr)
 392                 free_wrqe(wr);
 393
 394         if (toep) {
 395                 if (toep->tid >= 0)
 396                         free_atid(sc, toep->tid);
 397                 if (toep->l2te)
 398                         t4_l2t_release(toep->l2te);
 399                 if (toep->ce)
 400                         t4_release_lip(sc, toep->ce);
 401                 free_toepcb(toep);
 402         }
 403
 404         return (rc);
 405 }
 406 #endif