sys/dev/cxgbe/t4_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2018 Chelsio Communications, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/module.h>
  39 #include <sys/bus.h>
  40 #include <sys/lock.h>
  41 #include <sys/mutex.h>
  42 #include <sys/rwlock.h>
  43 #include <sys/socket.h>
  44 #include <sys/sbuf.h>
  45 #include <netinet/in.h>
  46
  47 #include "common/common.h"
  48 #include "common/t4_msg.h"
  49 #include "common/t4_regs.h"
  50 #include "common/t4_regs_values.h"
  51 #include "common/t4_tcb.h"
  52 #include "t4_l2t.h"
  53 #include "t4_smt.h"
  54
  55 struct filter_entry {
  56         uint32_t valid:1;       /* filter allocated and valid */
  57         uint32_t locked:1;      /* filter is administratively locked or busy */
  58         uint32_t pending:1;     /* filter action is pending firmware reply */
  59         int tid;                /* tid of the filter TCB */
  60         struct l2t_entry *l2te; /* L2 table entry for DMAC rewrite */
  61         struct smt_entry *smt;  /* SMT entry for SMAC rewrite */
  62
  63         struct t4_filter_specification fs;
  64 };
  65
  66 static void free_filter_resources(struct filter_entry *);
  67 static int get_hashfilter(struct adapter *, struct t4_filter *);
  68 static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t,
  69     struct l2t_entry *, struct smt_entry *);
  70 static int del_hashfilter(struct adapter *, struct t4_filter *);
  71 static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *);
  72
  73 static void
  74 insert_hftid(struct adapter *sc, int tid, void *ctx, int ntids)
  75 {
  76         struct tid_info *t = &sc->tids;
  77
  78         t->hftid_tab[tid] = ctx;
  79         atomic_add_int(&t->tids_in_use, ntids);
  80 }
  81
  82 static void *
  83 lookup_hftid(struct adapter *sc, int tid)
  84 {
  85         struct tid_info *t = &sc->tids;
  86
  87         return (t->hftid_tab[tid]);
  88 }
  89
  90 static void
  91 remove_hftid(struct adapter *sc, int tid, int ntids)
  92 {
  93         struct tid_info *t = &sc->tids;
  94
  95         t->hftid_tab[tid] = NULL;
  96         atomic_subtract_int(&t->tids_in_use, ntids);
  97 }
  98
  99 static uint32_t
 100 mode_to_fconf(uint32_t mode)
 101 {
 102         uint32_t fconf = 0;
 103
 104         if (mode & T4_FILTER_IP_FRAGMENT)
 105                 fconf |= F_FRAGMENTATION;
 106
 107         if (mode & T4_FILTER_MPS_HIT_TYPE)
 108                 fconf |= F_MPSHITTYPE;
 109
 110         if (mode & T4_FILTER_MAC_IDX)
 111                 fconf |= F_MACMATCH;
 112
 113         if (mode & T4_FILTER_ETH_TYPE)
 114                 fconf |= F_ETHERTYPE;
 115
 116         if (mode & T4_FILTER_IP_PROTO)
 117                 fconf |= F_PROTOCOL;
 118
 119         if (mode & T4_FILTER_IP_TOS)
 120                 fconf |= F_TOS;
 121
 122         if (mode & T4_FILTER_VLAN)
 123                 fconf |= F_VLAN;
 124
 125         if (mode & T4_FILTER_VNIC)
 126                 fconf |= F_VNIC_ID;
 127
 128         if (mode & T4_FILTER_PORT)
 129                 fconf |= F_PORT;
 130
 131         if (mode & T4_FILTER_FCoE)
 132                 fconf |= F_FCOE;
 133
 134         return (fconf);
 135 }
 136
 137 static uint32_t
 138 mode_to_iconf(uint32_t mode)
 139 {
 140
 141         if (mode & T4_FILTER_IC_VNIC)
 142                 return (F_VNIC);
 143         return (0);
 144 }
 145
 146 static int
 147 check_fspec_against_fconf_iconf(struct adapter *sc,
 148     struct t4_filter_specification *fs)
 149 {
 150         struct tp_params *tpp = &sc->params.tp;
 151         uint32_t fconf = 0;
 152
 153         if (fs->val.frag || fs->mask.frag)
 154                 fconf |= F_FRAGMENTATION;
 155
 156         if (fs->val.matchtype || fs->mask.matchtype)
 157                 fconf |= F_MPSHITTYPE;
 158
 159         if (fs->val.macidx || fs->mask.macidx)
 160                 fconf |= F_MACMATCH;
 161
 162         if (fs->val.ethtype || fs->mask.ethtype)
 163                 fconf |= F_ETHERTYPE;
 164
 165         if (fs->val.proto || fs->mask.proto)
 166                 fconf |= F_PROTOCOL;
 167
 168         if (fs->val.tos || fs->mask.tos)
 169                 fconf |= F_TOS;
 170
 171         if (fs->val.vlan_vld || fs->mask.vlan_vld)
 172                 fconf |= F_VLAN;
 173
 174         if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
 175                 fconf |= F_VNIC_ID;
 176                 if (tpp->ingress_config & F_VNIC)
 177                         return (EINVAL);
 178         }
 179
 180         if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
 181                 fconf |= F_VNIC_ID;
 182                 if ((tpp->ingress_config & F_VNIC) == 0)
 183                         return (EINVAL);
 184         }
 185
 186         if (fs->val.iport || fs->mask.iport)
 187                 fconf |= F_PORT;
 188
 189         if (fs->val.fcoe || fs->mask.fcoe)
 190                 fconf |= F_FCOE;
 191
 192         if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map)
 193                 return (E2BIG);
 194
 195         return (0);
 196 }
 197
 198 int
 199 get_filter_mode(struct adapter *sc, uint32_t *mode)
 200 {
 201         struct tp_params *tp = &sc->params.tp;
 202         uint64_t mask;
 203
 204         /* Non-zero incoming value in mode means "hashfilter mode". */
 205         mask = *mode ? tp->hash_filter_mask : UINT64_MAX;
 206
 207         /* Always */
 208         *mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
 209             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
 210
 211 #define CHECK_FIELD(fconf_bit, field_shift, field_mask, mode_bit)  do { \
 212         if (tp->vlan_pri_map & (fconf_bit)) { \
 213                 MPASS(tp->field_shift >= 0); \
 214                 if ((mask >> tp->field_shift & field_mask) == field_mask) \
 215                 *mode |= (mode_bit); \
 216         } \
 217 } while (0)
 218
 219         CHECK_FIELD(F_FRAGMENTATION, frag_shift, M_FT_FRAGMENTATION, T4_FILTER_IP_FRAGMENT);
 220         CHECK_FIELD(F_MPSHITTYPE, matchtype_shift, M_FT_MPSHITTYPE, T4_FILTER_MPS_HIT_TYPE);
 221         CHECK_FIELD(F_MACMATCH, macmatch_shift, M_FT_MACMATCH, T4_FILTER_MAC_IDX);
 222         CHECK_FIELD(F_ETHERTYPE, ethertype_shift, M_FT_ETHERTYPE, T4_FILTER_ETH_TYPE);
 223         CHECK_FIELD(F_PROTOCOL, protocol_shift, M_FT_PROTOCOL, T4_FILTER_IP_PROTO);
 224         CHECK_FIELD(F_TOS, tos_shift, M_FT_TOS, T4_FILTER_IP_TOS);
 225         CHECK_FIELD(F_VLAN, vlan_shift, M_FT_VLAN, T4_FILTER_VLAN);
 226         CHECK_FIELD(F_VNIC_ID, vnic_shift, M_FT_VNIC_ID , T4_FILTER_VNIC);
 227         if (tp->ingress_config & F_VNIC)
 228                 *mode |= T4_FILTER_IC_VNIC;
 229         CHECK_FIELD(F_PORT, port_shift, M_FT_PORT , T4_FILTER_PORT);
 230         CHECK_FIELD(F_FCOE, fcoe_shift, M_FT_FCOE , T4_FILTER_FCoE);
 231 #undef CHECK_FIELD
 232
 233         return (0);
 234 }
 235
 236 int
 237 set_filter_mode(struct adapter *sc, uint32_t mode)
 238 {
 239         struct tp_params *tpp = &sc->params.tp;
 240         uint32_t fconf, iconf;
 241         int rc;
 242
 243         iconf = mode_to_iconf(mode);
 244         if ((iconf ^ tpp->ingress_config) & F_VNIC) {
 245                 /*
 246                  * For now we just complain if A_TP_INGRESS_CONFIG is not
 247                  * already set to the correct value for the requested filter
 248                  * mode.  It's not clear if it's safe to write to this register
 249                  * on the fly.  (And we trust the cached value of the register).
 250                  *
 251                  * check_fspec_against_fconf_iconf and other code that looks at
 252                  * tp->vlan_pri_map and tp->ingress_config needs to be reviewed
 253                  * thorougly before allowing dynamic filter mode changes.
 254                  */
 255                 return (EBUSY);
 256         }
 257
 258         fconf = mode_to_fconf(mode);
 259
 260         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 261             "t4setfm");
 262         if (rc)
 263                 return (rc);
 264
 265         if (sc->tids.ftids_in_use > 0) {
 266                 rc = EBUSY;
 267                 goto done;
 268         }
 269
 270 #ifdef TCP_OFFLOAD
 271         if (uld_active(sc, ULD_TOM)) {
 272                 rc = EBUSY;
 273                 goto done;
 274         }
 275 #endif
 276
 277         rc = -t4_set_filter_mode(sc, fconf, true);
 278 done:
 279         end_synchronized_op(sc, LOCK_HELD);
 280         return (rc);
 281 }
 282
 283 static inline uint64_t
 284 get_filter_hits(struct adapter *sc, uint32_t tid)
 285 {
 286         uint32_t tcb_addr;
 287
 288         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE;
 289
 290         if (is_t4(sc)) {
 291                 uint64_t hits;
 292
 293                 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8);
 294                 return (be64toh(hits));
 295         } else {
 296                 uint32_t hits;
 297
 298                 read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4);
 299                 return (be32toh(hits));
 300         }
 301 }
 302
 303 int
 304 get_filter(struct adapter *sc, struct t4_filter *t)
 305 {
 306         int i, nfilters = sc->tids.nftids;
 307         struct filter_entry *f;
 308
 309         if (t->fs.hash)
 310                 return (get_hashfilter(sc, t));
 311
 312         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
 313             t->idx >= nfilters) {
 314                 t->idx = 0xffffffff;
 315                 return (0);
 316         }
 317
 318         mtx_lock(&sc->tids.ftid_lock);
 319         f = &sc->tids.ftid_tab[t->idx];
 320         for (i = t->idx; i < nfilters; i++, f++) {
 321                 if (f->valid) {
 322                         MPASS(f->tid == sc->tids.ftid_base + i);
 323                         t->idx = i;
 324                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
 325                         t->smtidx = f->smt ? f->smt->idx : 0;
 326                         if (f->fs.hitcnts)
 327                                 t->hits = get_filter_hits(sc, f->tid);
 328                         else
 329                                 t->hits = UINT64_MAX;
 330                         t->fs = f->fs;
 331
 332                         goto done;
 333                 }
 334         }
 335         t->idx = 0xffffffff;
 336 done:
 337         mtx_unlock(&sc->tids.ftid_lock);
 338         return (0);
 339 }
 340
 341 static int
 342 set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te,
 343     struct smt_entry *smt)
 344 {
 345         struct filter_entry *f;
 346         struct fw_filter2_wr *fwr;
 347         u_int vnic_vld, vnic_vld_mask;
 348         struct wrq_cookie cookie;
 349         int i, rc, busy, locked;
 350         const int ntids = t->fs.type ? 4 : 1;
 351
 352         MPASS(!t->fs.hash);
 353         MPASS(t->idx < sc->tids.nftids);
 354         /* Already validated against fconf, iconf */
 355         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
 356         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
 357
 358         f = &sc->tids.ftid_tab[t->idx];
 359         rc = busy = locked = 0;
 360         mtx_lock(&sc->tids.ftid_lock);
 361         for (i = 0; i < ntids; i++) {
 362                 busy += f[i].pending + f[i].valid;
 363                 locked += f[i].locked;
 364         }
 365         if (locked > 0)
 366                 rc = EPERM;
 367         else if (busy > 0)
 368                 rc = EBUSY;
 369         else {
 370                 int len16;
 371
 372                 if (sc->params.filter2_wr_support)
 373                         len16 = howmany(sizeof(struct fw_filter2_wr), 16);
 374                 else
 375                         len16 = howmany(sizeof(struct fw_filter_wr), 16);
 376                 fwr = start_wrq_wr(&sc->sge.mgmtq, len16, &cookie);
 377                 if (__predict_false(fwr == NULL))
 378                         rc = ENOMEM;
 379                 else {
 380                         f->pending = 1;
 381                         sc->tids.ftids_in_use++;
 382                 }
 383         }
 384         mtx_unlock(&sc->tids.ftid_lock);
 385         if (rc != 0) {
 386                 if (l2te)
 387                         t4_l2t_release(l2te);
 388                 if (smt)
 389                         t4_smt_release(smt);
 390                 return (rc);
 391         }
 392
 393         /*
 394          * Can't fail now.  A set-filter WR will definitely be sent.
 395          */
 396
 397         f->tid = sc->tids.ftid_base + t->idx;
 398         f->fs = t->fs;
 399         f->l2te = l2te;
 400         f->smt = smt;
 401
 402         if (t->fs.val.pfvf_vld || t->fs.val.ovlan_vld)
 403                 vnic_vld = 1;
 404         else
 405                 vnic_vld = 0;
 406         if (t->fs.mask.pfvf_vld || t->fs.mask.ovlan_vld)
 407                 vnic_vld_mask = 1;
 408         else
 409                 vnic_vld_mask = 0;
 410
 411         bzero(fwr, sizeof(*fwr));
 412         if (sc->params.filter2_wr_support)
 413                 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER2_WR));
 414         else
 415                 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
 416         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
 417         fwr->tid_to_iq =
 418             htobe32(V_FW_FILTER_WR_TID(f->tid) |
 419                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
 420                 V_FW_FILTER_WR_NOREPLY(0) |
 421                 V_FW_FILTER_WR_IQ(f->fs.iq));
 422         fwr->del_filter_to_l2tix =
 423             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
 424                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
 425                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
 426                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
 427                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
 428                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
 429                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
 430                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
 431                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
 432                     f->fs.newvlan == VLAN_REWRITE) |
 433                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
 434                     f->fs.newvlan == VLAN_REWRITE) |
 435                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
 436                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
 437                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
 438                 V_FW_FILTER_WR_L2TIX(f->l2te ? f->l2te->idx : 0));
 439         fwr->ethtype = htobe16(f->fs.val.ethtype);
 440         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
 441         fwr->frag_to_ovlan_vldm =
 442             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
 443                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
 444                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
 445                 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
 446                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
 447                 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
 448         fwr->smac_sel = 0;
 449         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
 450             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
 451         fwr->maci_to_matchtypem =
 452             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
 453                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
 454                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
 455                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
 456                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
 457                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
 458                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
 459                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
 460         fwr->ptcl = f->fs.val.proto;
 461         fwr->ptclm = f->fs.mask.proto;
 462         fwr->ttyp = f->fs.val.tos;
 463         fwr->ttypm = f->fs.mask.tos;
 464         fwr->ivlan = htobe16(f->fs.val.vlan);
 465         fwr->ivlanm = htobe16(f->fs.mask.vlan);
 466         fwr->ovlan = htobe16(f->fs.val.vnic);
 467         fwr->ovlanm = htobe16(f->fs.mask.vnic);
 468         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
 469         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
 470         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
 471         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
 472         fwr->lp = htobe16(f->fs.val.dport);
 473         fwr->lpm = htobe16(f->fs.mask.dport);
 474         fwr->fp = htobe16(f->fs.val.sport);
 475         fwr->fpm = htobe16(f->fs.mask.sport);
 476         /* sma = 0 tells the fw to use SMAC_SEL for source MAC address */
 477         bzero(fwr->sma, sizeof (fwr->sma));
 478         if (sc->params.filter2_wr_support) {
 479                 fwr->filter_type_swapmac =
 480                     V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac);
 481                 fwr->natmode_to_ulp_type =
 482                     V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ?
 483                         ULP_MODE_TCPDDP : ULP_MODE_NONE) |
 484                     V_FW_FILTER2_WR_NATFLAGCHECK(f->fs.nat_flag_chk) |
 485                     V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode);
 486                 memcpy(fwr->newlip, f->fs.nat_dip, sizeof(fwr->newlip));
 487                 memcpy(fwr->newfip, f->fs.nat_sip, sizeof(fwr->newfip));
 488                 fwr->newlport = htobe16(f->fs.nat_dport);
 489                 fwr->newfport = htobe16(f->fs.nat_sport);
 490                 fwr->natseqcheck = htobe32(f->fs.nat_seq_chk);
 491         }
 492         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
 493
 494         /* Wait for response. */
 495         mtx_lock(&sc->tids.ftid_lock);
 496         for (;;) {
 497                 if (f->pending == 0) {
 498                         rc = f->valid ? 0 : EIO;
 499                         break;
 500                 }
 501                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
 502                         rc = EINPROGRESS;
 503                         break;
 504                 }
 505         }
 506         mtx_unlock(&sc->tids.ftid_lock);
 507         return (rc);
 508 }
 509
 510 static int
 511 hashfilter_ntuple(struct adapter *sc, const struct t4_filter_specification *fs,
 512     uint64_t *ftuple)
 513 {
 514         struct tp_params *tp = &sc->params.tp;
 515         uint64_t fmask;
 516
 517         *ftuple = fmask = 0;
 518
 519         /*
 520          * Initialize each of the fields which we care about which are present
 521          * in the Compressed Filter Tuple.
 522          */
 523         if (tp->vlan_shift >= 0 && fs->mask.vlan) {
 524                 *ftuple |= (F_FT_VLAN_VLD | fs->val.vlan) << tp->vlan_shift;
 525                 fmask |= M_FT_VLAN << tp->vlan_shift;
 526         }
 527
 528         if (tp->port_shift >= 0 && fs->mask.iport) {
 529                 *ftuple |= (uint64_t)fs->val.iport << tp->port_shift;
 530                 fmask |= M_FT_PORT << tp->port_shift;
 531         }
 532
 533         if (tp->protocol_shift >= 0 && fs->mask.proto) {
 534                 *ftuple |= (uint64_t)fs->val.proto << tp->protocol_shift;
 535                 fmask |= M_FT_PROTOCOL << tp->protocol_shift;
 536         }
 537
 538         if (tp->tos_shift >= 0 && fs->mask.tos) {
 539                 *ftuple |= (uint64_t)(fs->val.tos) << tp->tos_shift;
 540                 fmask |= M_FT_TOS << tp->tos_shift;
 541         }
 542
 543         if (tp->vnic_shift >= 0 && fs->mask.vnic) {
 544                 /* F_VNIC in ingress config was already validated. */
 545                 if (tp->ingress_config & F_VNIC)
 546                         MPASS(fs->mask.pfvf_vld);
 547                 else
 548                         MPASS(fs->mask.ovlan_vld);
 549
 550                 *ftuple |= ((1ULL << 16) | fs->val.vnic) << tp->vnic_shift;
 551                 fmask |= M_FT_VNIC_ID << tp->vnic_shift;
 552         }
 553
 554         if (tp->macmatch_shift >= 0 && fs->mask.macidx) {
 555                 *ftuple |= (uint64_t)(fs->val.macidx) << tp->macmatch_shift;
 556                 fmask |= M_FT_MACMATCH << tp->macmatch_shift;
 557         }
 558
 559         if (tp->ethertype_shift >= 0 && fs->mask.ethtype) {
 560                 *ftuple |= (uint64_t)(fs->val.ethtype) << tp->ethertype_shift;
 561                 fmask |= M_FT_ETHERTYPE << tp->ethertype_shift;
 562         }
 563
 564         if (tp->matchtype_shift >= 0 && fs->mask.matchtype) {
 565                 *ftuple |= (uint64_t)(fs->val.matchtype) << tp->matchtype_shift;
 566                 fmask |= M_FT_MPSHITTYPE << tp->matchtype_shift;
 567         }
 568
 569         if (tp->frag_shift >= 0 && fs->mask.frag) {
 570                 *ftuple |= (uint64_t)(fs->val.frag) << tp->frag_shift;
 571                 fmask |= M_FT_FRAGMENTATION << tp->frag_shift;
 572         }
 573
 574         if (tp->fcoe_shift >= 0 && fs->mask.fcoe) {
 575                 *ftuple |= (uint64_t)(fs->val.fcoe) << tp->fcoe_shift;
 576                 fmask |= M_FT_FCOE << tp->fcoe_shift;
 577         }
 578
 579         /* A hashfilter must conform to the filterMask. */
 580         if (fmask != tp->hash_filter_mask)
 581                 return (EINVAL);
 582
 583         return (0);
 584 }
 585
 586 int
 587 set_filter(struct adapter *sc, struct t4_filter *t)
 588 {
 589         struct tid_info *ti = &sc->tids;
 590         struct l2t_entry *l2te;
 591         struct smt_entry *smt;
 592         uint64_t ftuple;
 593         int rc;
 594
 595         /*
 596          * Basic filter checks first.
 597          */
 598
 599         if (t->fs.hash) {
 600                 if (!is_hashfilter(sc) || ti->ntids == 0)
 601                         return (ENOTSUP);
 602                 /* Hardware, not user, selects a tid for hashfilters. */
 603                 if (t->idx != (uint32_t)-1)
 604                         return (EINVAL);
 605                 /* T5 can't count hashfilter hits. */
 606                 if (is_t5(sc) && t->fs.hitcnts)
 607                         return (EINVAL);
 608                 rc = hashfilter_ntuple(sc, &t->fs, &ftuple);
 609                 if (rc != 0)
 610                         return (rc);
 611         } else {
 612                 if (ti->nftids == 0)
 613                         return (ENOTSUP);
 614                 if (t->idx >= ti->nftids)
 615                         return (EINVAL);
 616                 /* IPv6 filter idx must be 4 aligned */
 617                 if (t->fs.type == 1 &&
 618                     ((t->idx & 0x3) || t->idx + 4 >= ti->nftids))
 619                         return (EINVAL);
 620         }
 621
 622         /* T4 doesn't support VLAN tag removal or rewrite, swapmac, and NAT. */
 623         if (is_t4(sc) && t->fs.action == FILTER_SWITCH &&
 624             (t->fs.newvlan == VLAN_REMOVE || t->fs.newvlan == VLAN_REWRITE ||
 625             t->fs.swapmac || t->fs.nat_mode))
 626                 return (ENOTSUP);
 627
 628         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= sc->params.nports)
 629                 return (EINVAL);
 630         if (t->fs.val.iport >= sc->params.nports)
 631                 return (EINVAL);
 632
 633         /* Can't specify an iq if not steering to it */
 634         if (!t->fs.dirsteer && t->fs.iq)
 635                 return (EINVAL);
 636
 637         /* Validate against the global filter mode and ingress config */
 638         rc = check_fspec_against_fconf_iconf(sc, &t->fs);
 639         if (rc != 0)
 640                 return (rc);
 641
 642         /*
 643          * Basic checks passed.  Make sure the queues and tid tables are setup.
 644          */
 645
 646         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
 647         if (rc)
 648                 return (rc);
 649         if (!(sc->flags & FULL_INIT_DONE) &&
 650             ((rc = adapter_full_init(sc)) != 0)) {
 651                 end_synchronized_op(sc, 0);
 652                 return (rc);
 653         }
 654         if (t->fs.hash) {
 655                 if (__predict_false(ti->hftid_tab == NULL)) {
 656                         ti->hftid_tab = malloc(sizeof(*ti->hftid_tab) * ti->ntids,
 657                             M_CXGBE, M_NOWAIT | M_ZERO);
 658                         if (ti->hftid_tab == NULL) {
 659                                 rc = ENOMEM;
 660                                 goto done;
 661                         }
 662                         mtx_init(&ti->hftid_lock, "T4 hashfilters", 0, MTX_DEF);
 663                         cv_init(&ti->hftid_cv, "t4hfcv");
 664                 }
 665                 if (__predict_false(sc->tids.atid_tab == NULL)) {
 666                         rc = alloc_atid_tab(&sc->tids, M_NOWAIT);
 667                         if (rc != 0)
 668                                 goto done;
 669                 }
 670         } else if (__predict_false(ti->ftid_tab == NULL)) {
 671                 KASSERT(ti->ftids_in_use == 0,
 672                     ("%s: no memory allocated but ftids_in_use > 0", __func__));
 673                 ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids,
 674                     M_CXGBE, M_NOWAIT | M_ZERO);
 675                 if (ti->ftid_tab == NULL) {
 676                         rc = ENOMEM;
 677                         goto done;
 678                 }
 679                 mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
 680                 cv_init(&ti->ftid_cv, "t4fcv");
 681         }
 682 done:
 683         end_synchronized_op(sc, 0);
 684         if (rc != 0)
 685                 return (rc);
 686
 687         /*
 688          * Allocate L2T entry, SMT entry, etc.
 689          */
 690
 691         l2te = NULL;
 692         if (t->fs.newdmac || t->fs.newvlan) {
 693                 /* This filter needs an L2T entry; allocate one. */
 694                 l2te = t4_l2t_alloc_switching(sc->l2t);
 695                 if (__predict_false(l2te == NULL))
 696                         return (EAGAIN);
 697                 rc = t4_l2t_set_switching(sc, l2te, t->fs.vlan, t->fs.eport,
 698                     t->fs.dmac);
 699                 if (rc) {
 700                         t4_l2t_release(l2te);
 701                         return (ENOMEM);
 702                 }
 703         }
 704
 705         smt = NULL;
 706         if (t->fs.newsmac) {
 707                 /* This filter needs an SMT entry; allocate one. */
 708                 smt = t4_smt_alloc_switching(sc->smt, t->fs.smac);
 709                 if (__predict_false(smt == NULL)) {
 710                         if (l2te != NULL)
 711                                 t4_l2t_release(l2te);
 712                         return (EAGAIN);
 713                 }
 714                 rc = t4_smt_set_switching(sc, smt, 0x0, t->fs.smac);
 715                 if (rc) {
 716                         t4_smt_release(smt);
 717                         if (l2te != NULL)
 718                                 t4_l2t_release(l2te);
 719                         return (rc);
 720                 }
 721         }
 722
 723         if (t->fs.hash)
 724                 return (set_hashfilter(sc, t, ftuple, l2te, smt));
 725         else
 726                 return (set_tcamfilter(sc, t, l2te, smt));
 727
 728 }
 729
 730 static int
 731 del_tcamfilter(struct adapter *sc, struct t4_filter *t)
 732 {
 733         struct filter_entry *f;
 734         struct fw_filter_wr *fwr;
 735         struct wrq_cookie cookie;
 736         int rc;
 737
 738         MPASS(sc->tids.ftid_tab != NULL);
 739         MPASS(sc->tids.nftids > 0);
 740
 741         if (t->idx >= sc->tids.nftids)
 742                 return (EINVAL);
 743
 744         mtx_lock(&sc->tids.ftid_lock);
 745         f = &sc->tids.ftid_tab[t->idx];
 746         if (f->locked) {
 747                 rc = EPERM;
 748                 goto done;
 749         }
 750         if (f->pending) {
 751                 rc = EBUSY;
 752                 goto done;
 753         }
 754         if (f->valid == 0) {
 755                 rc = EINVAL;
 756                 goto done;
 757         }
 758         MPASS(f->tid == sc->tids.ftid_base + t->idx);
 759         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
 760         if (fwr == NULL) {
 761                 rc = ENOMEM;
 762                 goto done;
 763         }
 764
 765         bzero(fwr, sizeof (*fwr));
 766         t4_mk_filtdelwr(f->tid, fwr, sc->sge.fwq.abs_id);
 767         f->pending = 1;
 768         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
 769         t->fs = f->fs;  /* extra info for the caller */
 770
 771         for (;;) {
 772                 if (f->pending == 0) {
 773                         rc = f->valid ? EIO : 0;
 774                         break;
 775                 }
 776                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
 777                         rc = EINPROGRESS;
 778                         break;
 779                 }
 780         }
 781 done:
 782         mtx_unlock(&sc->tids.ftid_lock);
 783         return (rc);
 784 }
 785
 786 int
 787 del_filter(struct adapter *sc, struct t4_filter *t)
 788 {
 789
 790         /* No filters possible if not initialized yet. */
 791         if (!(sc->flags & FULL_INIT_DONE))
 792                 return (EINVAL);
 793
 794         /*
 795          * The checks for tid tables ensure that the locks that del_* will reach
 796          * for are initialized.
 797          */
 798         if (t->fs.hash) {
 799                 if (sc->tids.hftid_tab != NULL)
 800                         return (del_hashfilter(sc, t));
 801         } else {
 802                 if (sc->tids.ftid_tab != NULL)
 803                         return (del_tcamfilter(sc, t));
 804         }
 805
 806         return (EINVAL);
 807 }
 808
 809 /*
 810  * Release secondary resources associated with the filter.
 811  */
 812 static void
 813 free_filter_resources(struct filter_entry *f)
 814 {
 815
 816         if (f->l2te) {
 817                 t4_l2t_release(f->l2te);
 818                 f->l2te = NULL;
 819         }
 820         if (f->smt) {
 821                 t4_smt_release(f->smt);
 822                 f->smt = NULL;
 823         }
 824 }
 825
 826 static int
 827 set_tcb_field(struct adapter *sc, u_int tid, uint16_t word, uint64_t mask,
 828     uint64_t val, int no_reply)
 829 {
 830         struct wrq_cookie cookie;
 831         struct cpl_set_tcb_field *req;
 832
 833         req = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*req), 16), &cookie);
 834         if (req == NULL)
 835                 return (ENOMEM);
 836         bzero(req, sizeof(*req));
 837         INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid);
 838         if (no_reply == 0) {
 839                 req->reply_ctrl = htobe16(V_QUEUENO(sc->sge.fwq.abs_id) |
 840                     V_NO_REPLY(0));
 841         } else
 842                 req->reply_ctrl = htobe16(V_NO_REPLY(1));
 843         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(CPL_COOKIE_HASHFILTER));
 844         req->mask = htobe64(mask);
 845         req->val = htobe64(val);
 846         commit_wrq_wr(&sc->sge.mgmtq, req, &cookie);
 847
 848         return (0);
 849 }
 850
 851 /* Set one of the t_flags bits in the TCB. */
 852 static inline int
 853 set_tcb_tflag(struct adapter *sc, int tid, u_int bit_pos, u_int val,
 854     u_int no_reply)
 855 {
 856
 857         return (set_tcb_field(sc, tid,  W_TCB_T_FLAGS, 1ULL << bit_pos,
 858             (uint64_t)val << bit_pos, no_reply));
 859 }
 860
 861 int
 862 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 863 {
 864         struct adapter *sc = iq->adapter;
 865         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
 866         u_int tid = GET_TID(rpl);
 867         u_int rc, cleanup, idx;
 868         struct filter_entry *f;
 869
 870         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
 871             rss->opcode));
 872         MPASS(is_ftid(sc, tid));
 873
 874         cleanup = 0;
 875         idx = tid - sc->tids.ftid_base;
 876         f = &sc->tids.ftid_tab[idx];
 877         rc = G_COOKIE(rpl->cookie);
 878
 879         mtx_lock(&sc->tids.ftid_lock);
 880         KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.",
 881             __func__, rc, idx));
 882         switch(rc) {
 883         case FW_FILTER_WR_FLT_ADDED:
 884                 /* set-filter succeeded */
 885                 f->valid = 1;
 886                 if (f->fs.newsmac) {
 887                         MPASS(f->smt != NULL);
 888                         set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
 889                         set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
 890                             V_TCB_SMAC_SEL(M_TCB_SMAC_SEL),
 891                             V_TCB_SMAC_SEL(f->smt->idx), 1);
 892                         /* XXX: wait for reply to TCB update before !pending */
 893                 }
 894                 break;
 895         case FW_FILTER_WR_FLT_DELETED:
 896                 /* del-filter succeeded */
 897                 MPASS(f->valid == 1);
 898                 f->valid = 0;
 899                 /* Fall through */
 900         case FW_FILTER_WR_SMT_TBL_FULL:
 901                 /* set-filter failed due to lack of SMT space. */
 902                 MPASS(f->valid == 0);
 903                 free_filter_resources(f);
 904                 sc->tids.ftids_in_use--;
 905                 break;
 906         case FW_FILTER_WR_SUCCESS:
 907         case FW_FILTER_WR_EINVAL:
 908         default:
 909                 panic("%s: unexpected reply %d for filter[%d].", __func__, rc,
 910                     idx);
 911         }
 912         f->pending = 0;
 913         cv_broadcast(&sc->tids.ftid_cv);
 914         mtx_unlock(&sc->tids.ftid_lock);
 915
 916         return (0);
 917 }
 918
 919 /*
 920  * This is the reply to the Active Open that created the filter.  Additional TCB
 921  * updates may be required to complete the filter configuration.
 922  */
 923 int
 924 t4_hashfilter_ao_rpl(struct sge_iq *iq, const struct rss_header *rss,
 925     struct mbuf *m)
 926 {
 927         struct adapter *sc = iq->adapter;
 928         const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
 929         u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
 930         u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
 931         struct filter_entry *f = lookup_atid(sc, atid);
 932
 933         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 934
 935         mtx_lock(&sc->tids.hftid_lock);
 936         KASSERT(f->pending, ("%s: hashfilter[%p] isn't pending.", __func__, f));
 937         KASSERT(f->tid == -1, ("%s: hashfilter[%p] has tid %d already.",
 938             __func__, f, f->tid));
 939         if (status == CPL_ERR_NONE) {
 940                 struct filter_entry *f2;
 941
 942                 f->tid = GET_TID(cpl);
 943                 MPASS(f->tid < sc->tids.ntids);
 944                 if (__predict_false((f2 = lookup_hftid(sc, f->tid)) != NULL)) {
 945                         /* XXX: avoid hash collisions in the first place. */
 946                         MPASS(f2->tid == f->tid);
 947                         remove_hftid(sc, f2->tid, f2->fs.type ? 2 : 1);
 948                         free_filter_resources(f2);
 949                         free(f2, M_CXGBE);
 950                 }
 951                 insert_hftid(sc, f->tid, f, f->fs.type ? 2 : 1);
 952                 /*
 953                  * Leave the filter pending until it is fully set up, which will
 954                  * be indicated by the reply to the last TCB update.  No need to
 955                  * unblock the ioctl thread either.
 956                  */
 957                 if (configure_hashfilter_tcb(sc, f) == EINPROGRESS)
 958                         goto done;
 959                 f->valid = 1;
 960                 f->pending = 0;
 961         } else {
 962                 /* provide errno instead of tid to ioctl */
 963                 f->tid = act_open_rpl_status_to_errno(status);
 964                 f->valid = 0;
 965                 if (act_open_has_tid(status))
 966                         release_tid(sc, GET_TID(cpl), &sc->sge.mgmtq);
 967                 free_filter_resources(f);
 968                 if (f->locked == 0)
 969                         free(f, M_CXGBE);
 970         }
 971         cv_broadcast(&sc->tids.hftid_cv);
 972 done:
 973         mtx_unlock(&sc->tids.hftid_lock);
 974
 975         free_atid(sc, atid);
 976         return (0);
 977 }
 978
 979 int
 980 t4_hashfilter_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss,
 981     struct mbuf *m)
 982 {
 983         struct adapter *sc = iq->adapter;
 984         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
 985         u_int tid = GET_TID(rpl);
 986         struct filter_entry *f;
 987
 988         mtx_lock(&sc->tids.hftid_lock);
 989         f = lookup_hftid(sc, tid);
 990         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
 991         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
 992             f, tid));
 993         KASSERT(f->valid == 0, ("%s: hashfilter %p [%u] is valid already.",
 994             __func__, f, tid));
 995         f->pending = 0;
 996         if (rpl->status == 0) {
 997                 f->valid = 1;
 998         } else {
 999                 f->tid = EIO;
1000                 f->valid = 0;
1001                 free_filter_resources(f);
1002                 remove_hftid(sc, tid, f->fs.type ? 2 : 1);
1003                 release_tid(sc, tid, &sc->sge.mgmtq);
1004                 if (f->locked == 0)
1005                         free(f, M_CXGBE);
1006         }
1007         cv_broadcast(&sc->tids.hftid_cv);
1008         mtx_unlock(&sc->tids.hftid_lock);
1009
1010         return (0);
1011 }
1012
1013 int
1014 t4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss,
1015     struct mbuf *m)
1016 {
1017         struct adapter *sc = iq->adapter;
1018         const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1019         unsigned int tid = GET_TID(cpl);
1020         struct filter_entry *f;
1021
1022         mtx_lock(&sc->tids.hftid_lock);
1023         f = lookup_hftid(sc, tid);
1024         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
1025         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
1026             f, tid));
1027         KASSERT(f->valid, ("%s: hashfilter %p [%u] isn't valid.", __func__, f,
1028             tid));
1029         f->pending = 0;
1030         if (cpl->status == 0) {
1031                 f->valid = 0;
1032                 free_filter_resources(f);
1033                 remove_hftid(sc, tid, f->fs.type ? 2 : 1);
1034                 release_tid(sc, tid, &sc->sge.mgmtq);
1035                 if (f->locked == 0)
1036                         free(f, M_CXGBE);
1037         }
1038         cv_broadcast(&sc->tids.hftid_cv);
1039         mtx_unlock(&sc->tids.hftid_lock);
1040
1041         return (0);
1042 }
1043
1044 static int
1045 get_hashfilter(struct adapter *sc, struct t4_filter *t)
1046 {
1047         int i, nfilters = sc->tids.ntids;
1048         struct filter_entry *f;
1049
1050         if (sc->tids.tids_in_use == 0 || sc->tids.hftid_tab == NULL ||
1051             t->idx >= nfilters) {
1052                 t->idx = 0xffffffff;
1053                 return (0);
1054         }
1055
1056         mtx_lock(&sc->tids.hftid_lock);
1057         for (i = t->idx; i < nfilters; i++) {
1058                 f = lookup_hftid(sc, i);
1059                 if (f != NULL && f->valid) {
1060                         t->idx = i;
1061                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
1062                         t->smtidx = f->smt ? f->smt->idx : 0;
1063                         if (f->fs.hitcnts)
1064                                 t->hits = get_filter_hits(sc, t->idx);
1065                         else
1066                                 t->hits = UINT64_MAX;
1067                         t->fs = f->fs;
1068
1069                         goto done;
1070                 }
1071         }
1072         t->idx = 0xffffffff;
1073 done:
1074         mtx_unlock(&sc->tids.hftid_lock);
1075         return (0);
1076 }
1077
1078 static void
1079 mk_act_open_req6(struct adapter *sc, struct filter_entry *f, int atid,
1080     uint64_t ftuple, struct cpl_act_open_req6 *cpl)
1081 {
1082         struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
1083         struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
1084
1085         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1086         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1087         MPASS(atid >= 0);
1088
1089         if (chip_id(sc) == CHELSIO_T5) {
1090                 INIT_TP_WR(cpl5, 0);
1091         } else {
1092                 INIT_TP_WR(cpl6, 0);
1093                 cpl6->rsvd2 = 0;
1094                 cpl6->opt3 = 0;
1095         }
1096
1097         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
1098             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1099             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1100         cpl->local_port = htobe16(f->fs.val.dport);
1101         cpl->peer_port = htobe16(f->fs.val.sport);
1102         cpl->local_ip_hi = *(uint64_t *)(&f->fs.val.dip);
1103         cpl->local_ip_lo = *(((uint64_t *)&f->fs.val.dip) + 1);
1104         cpl->peer_ip_hi = *(uint64_t *)(&f->fs.val.sip);
1105         cpl->peer_ip_lo = *(((uint64_t *)&f->fs.val.sip) + 1);
1106         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1107             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1108             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1109             V_NO_CONG(f->fs.rpttid) |
1110             V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1111             F_TCAM_BYPASS | F_NON_OFFLOAD);
1112
1113         cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1114         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1115             V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1116             V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1117             F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1118             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1119             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1120 }
1121
1122 static void
1123 mk_act_open_req(struct adapter *sc, struct filter_entry *f, int atid,
1124     uint64_t ftuple, struct cpl_act_open_req *cpl)
1125 {
1126         struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
1127         struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
1128
1129         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1130         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1131         MPASS(atid >= 0);
1132
1133         if (chip_id(sc) == CHELSIO_T5) {
1134                 INIT_TP_WR(cpl5, 0);
1135         } else {
1136                 INIT_TP_WR(cpl6, 0);
1137                 cpl6->rsvd2 = 0;
1138                 cpl6->opt3 = 0;
1139         }
1140
1141         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
1142             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1143             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1144         cpl->local_port = htobe16(f->fs.val.dport);
1145         cpl->peer_port = htobe16(f->fs.val.sport);
1146         cpl->local_ip = f->fs.val.dip[0] | f->fs.val.dip[1] << 8 |
1147             f->fs.val.dip[2] << 16 | f->fs.val.dip[3] << 24;
1148         cpl->peer_ip = f->fs.val.sip[0] | f->fs.val.sip[1] << 8 |
1149                 f->fs.val.sip[2] << 16 | f->fs.val.sip[3] << 24;
1150         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1151             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1152             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1153             V_NO_CONG(f->fs.rpttid) |
1154             V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1155             F_TCAM_BYPASS | F_NON_OFFLOAD);
1156
1157         cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1158         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1159             V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1160             V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1161             F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1162             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1163             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1164 }
1165
1166 static int
1167 act_open_cpl_len16(struct adapter *sc, int isipv6)
1168 {
1169         int idx;
1170         static const int sz_table[3][2] = {
1171                 {
1172                         howmany(sizeof (struct cpl_act_open_req), 16),
1173                         howmany(sizeof (struct cpl_act_open_req6), 16)
1174                 },
1175                 {
1176                         howmany(sizeof (struct cpl_t5_act_open_req), 16),
1177                         howmany(sizeof (struct cpl_t5_act_open_req6), 16)
1178                 },
1179                 {
1180                         howmany(sizeof (struct cpl_t6_act_open_req), 16),
1181                         howmany(sizeof (struct cpl_t6_act_open_req6), 16)
1182                 },
1183         };
1184
1185         MPASS(chip_id(sc) >= CHELSIO_T4);
1186         idx = min(chip_id(sc) - CHELSIO_T4, 2);
1187
1188         return (sz_table[idx][!!isipv6]);
1189 }
1190
1191 static int
1192 set_hashfilter(struct adapter *sc, struct t4_filter *t, uint64_t ftuple,
1193     struct l2t_entry *l2te, struct smt_entry *smt)
1194 {
1195         void *wr;
1196         struct wrq_cookie cookie;
1197         struct filter_entry *f;
1198         int rc, atid = -1;
1199
1200         MPASS(t->fs.hash);
1201         /* Already validated against fconf, iconf */
1202         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
1203         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
1204
1205         mtx_lock(&sc->tids.hftid_lock);
1206
1207         /*
1208          * XXX: Check for hash collisions and insert in the hash based lookup
1209          * table so that in-flight hashfilters are also considered when checking
1210          * for collisions.
1211          */
1212
1213         f = malloc(sizeof(*f), M_CXGBE, M_ZERO | M_NOWAIT);
1214         if (__predict_false(f == NULL)) {
1215                 if (l2te)
1216                         t4_l2t_release(l2te);
1217                 if (smt)
1218                         t4_smt_release(smt);
1219                 rc = ENOMEM;
1220                 goto done;
1221         }
1222         f->fs = t->fs;
1223         f->l2te = l2te;
1224         f->smt = smt;
1225
1226         atid = alloc_atid(sc, f);
1227         if (__predict_false(atid) == -1) {
1228                 if (l2te)
1229                         t4_l2t_release(l2te);
1230                 if (smt)
1231                         t4_smt_release(smt);
1232                 free(f, M_CXGBE);
1233                 rc = EAGAIN;
1234                 goto done;
1235         }
1236         MPASS(atid >= 0);
1237
1238         wr = start_wrq_wr(&sc->sge.mgmtq, act_open_cpl_len16(sc, f->fs.type),
1239             &cookie);
1240         if (wr == NULL) {
1241                 free_atid(sc, atid);
1242                 if (l2te)
1243                         t4_l2t_release(l2te);
1244                 if (smt)
1245                         t4_smt_release(smt);
1246                 free(f, M_CXGBE);
1247                 rc = ENOMEM;
1248                 goto done;
1249         }
1250         if (f->fs.type)
1251                 mk_act_open_req6(sc, f, atid, ftuple, wr);
1252         else
1253                 mk_act_open_req(sc, f, atid, ftuple, wr);
1254
1255         f->locked = 1; /* ithread mustn't free f if ioctl is still around. */
1256         f->pending = 1;
1257         f->tid = -1;
1258         commit_wrq_wr(&sc->sge.mgmtq, wr, &cookie);
1259
1260         for (;;) {
1261                 MPASS(f->locked);
1262                 if (f->pending == 0) {
1263                         if (f->valid) {
1264                                 rc = 0;
1265                                 f->locked = 0;
1266                                 t->idx = f->tid;
1267                         } else {
1268                                 rc = f->tid;
1269                                 free(f, M_CXGBE);
1270                         }
1271                         break;
1272                 }
1273                 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1274                         f->locked = 0;
1275                         rc = EINPROGRESS;
1276                         break;
1277                 }
1278         }
1279 done:
1280         mtx_unlock(&sc->tids.hftid_lock);
1281         return (rc);
1282 }
1283
1284 /* SET_TCB_FIELD sent as a ULP command looks like this */
1285 #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \
1286     sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core))
1287
1288 static void *
1289 mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, uint64_t word, uint64_t mask,
1290                 uint64_t val, uint32_t tid, uint32_t qid)
1291 {
1292         struct ulptx_idata *ulpsc;
1293         struct cpl_set_tcb_field_core *req;
1294
1295         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1296         ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
1297
1298         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1299         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1300         ulpsc->len = htobe32(sizeof(*req));
1301
1302         req = (struct cpl_set_tcb_field_core *)(ulpsc + 1);
1303         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1304         req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(qid));
1305         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
1306         req->mask = htobe64(mask);
1307         req->val = htobe64(val);
1308
1309         ulpsc = (struct ulptx_idata *)(req + 1);
1310         if (LEN__SET_TCB_FIELD_ULP % 16) {
1311                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1312                 ulpsc->len = htobe32(0);
1313                 return (ulpsc + 1);
1314         }
1315         return (ulpsc);
1316 }
1317
1318 /* ABORT_REQ sent as a ULP command looks like this */
1319 #define LEN__ABORT_REQ_ULP (sizeof(struct ulp_txpkt) + \
1320         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_req_core))
1321
1322 static void *
1323 mk_abort_req_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1324 {
1325         struct ulptx_idata *ulpsc;
1326         struct cpl_abort_req_core *req;
1327
1328         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1329         ulpmc->len = htobe32(howmany(LEN__ABORT_REQ_ULP, 16));
1330
1331         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1332         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1333         ulpsc->len = htobe32(sizeof(*req));
1334
1335         req = (struct cpl_abort_req_core *)(ulpsc + 1);
1336         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
1337         req->rsvd0 = htonl(0);
1338         req->rsvd1 = 0;
1339         req->cmd = CPL_ABORT_NO_RST;
1340
1341         ulpsc = (struct ulptx_idata *)(req + 1);
1342         if (LEN__ABORT_REQ_ULP % 16) {
1343                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1344                 ulpsc->len = htobe32(0);
1345                 return (ulpsc + 1);
1346         }
1347         return (ulpsc);
1348 }
1349
1350 /* ABORT_RPL sent as a ULP command looks like this */
1351 #define LEN__ABORT_RPL_ULP (sizeof(struct ulp_txpkt) + \
1352         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_rpl_core))
1353
1354 static void *
1355 mk_abort_rpl_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1356 {
1357         struct ulptx_idata *ulpsc;
1358         struct cpl_abort_rpl_core *rpl;
1359
1360         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1361         ulpmc->len = htobe32(howmany(LEN__ABORT_RPL_ULP, 16));
1362
1363         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1364         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1365         ulpsc->len = htobe32(sizeof(*rpl));
1366
1367         rpl = (struct cpl_abort_rpl_core *)(ulpsc + 1);
1368         OPCODE_TID(rpl) = htobe32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
1369         rpl->rsvd0 = htonl(0);
1370         rpl->rsvd1 = 0;
1371         rpl->cmd = CPL_ABORT_NO_RST;
1372
1373         ulpsc = (struct ulptx_idata *)(rpl + 1);
1374         if (LEN__ABORT_RPL_ULP % 16) {
1375                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1376                 ulpsc->len = htobe32(0);
1377                 return (ulpsc + 1);
1378         }
1379         return (ulpsc);
1380 }
1381
1382 static inline int
1383 del_hashfilter_wrlen(void)
1384 {
1385
1386         return (sizeof(struct work_request_hdr) +
1387             roundup2(LEN__SET_TCB_FIELD_ULP, 16) +
1388             roundup2(LEN__ABORT_REQ_ULP, 16) +
1389             roundup2(LEN__ABORT_RPL_ULP, 16));
1390 }
1391
1392 static void
1393 mk_del_hashfilter_wr(int tid, struct work_request_hdr *wrh, int wrlen, int qid)
1394 {
1395         struct ulp_txpkt *ulpmc;
1396
1397         INIT_ULPTX_WRH(wrh, wrlen, 0, 0);
1398         ulpmc = (struct ulp_txpkt *)(wrh + 1);
1399         ulpmc = mk_set_tcb_field_ulp(ulpmc, W_TCB_RSS_INFO,
1400             V_TCB_RSS_INFO(M_TCB_RSS_INFO), V_TCB_RSS_INFO(qid), tid, 0);
1401         ulpmc = mk_abort_req_ulp(ulpmc, tid);
1402         ulpmc = mk_abort_rpl_ulp(ulpmc, tid);
1403 }
1404
1405 static int
1406 del_hashfilter(struct adapter *sc, struct t4_filter *t)
1407 {
1408         void *wr;
1409         struct filter_entry *f;
1410         struct wrq_cookie cookie;
1411         int rc;
1412         const int wrlen = del_hashfilter_wrlen();
1413
1414         MPASS(sc->tids.hftid_tab != NULL);
1415         MPASS(sc->tids.ntids > 0);
1416
1417         if (t->idx >= sc->tids.ntids)
1418                 return (EINVAL);
1419
1420         mtx_lock(&sc->tids.hftid_lock);
1421         f = lookup_hftid(sc, t->idx);
1422         if (f == NULL || f->valid == 0) {
1423                 rc = EINVAL;
1424                 goto done;
1425         }
1426         MPASS(f->tid == t->idx);
1427         if (f->locked) {
1428                 rc = EPERM;
1429                 goto done;
1430         }
1431         if (f->pending) {
1432                 rc = EBUSY;
1433                 goto done;
1434         }
1435         wr = start_wrq_wr(&sc->sge.mgmtq, howmany(wrlen, 16), &cookie);
1436         if (wr == NULL) {
1437                 rc = ENOMEM;
1438                 goto done;
1439         }
1440
1441         mk_del_hashfilter_wr(t->idx, wr, wrlen, sc->sge.fwq.abs_id);
1442         f->locked = 1;
1443         f->pending = 1;
1444         commit_wrq_wr(&sc->sge.mgmtq, wr, &cookie);
1445         t->fs = f->fs;  /* extra info for the caller */
1446
1447         for (;;) {
1448                 MPASS(f->locked);
1449                 if (f->pending == 0) {
1450                         if (f->valid) {
1451                                 f->locked = 0;
1452                                 rc = EIO;
1453                         } else {
1454                                 rc = 0;
1455                                 free(f, M_CXGBE);
1456                         }
1457                         break;
1458                 }
1459                 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1460                         f->locked = 0;
1461                         rc = EINPROGRESS;
1462                         break;
1463                 }
1464         }
1465 done:
1466         mtx_unlock(&sc->tids.hftid_lock);
1467         return (rc);
1468 }
1469
1470 #define WORD_MASK       0xffffffff
1471 static void
1472 set_nat_params(struct adapter *sc, struct filter_entry *f, const bool dip,
1473     const bool sip, const bool dp, const bool sp)
1474 {
1475
1476         if (dip) {
1477                 if (f->fs.type) {
1478                         set_tcb_field(sc, f->tid, W_TCB_SND_UNA_RAW, WORD_MASK,
1479                             f->fs.nat_dip[15] | f->fs.nat_dip[14] << 8 |
1480                             f->fs.nat_dip[13] << 16 | f->fs.nat_dip[12] << 24, 1);
1481
1482                         set_tcb_field(sc, f->tid,
1483                             W_TCB_SND_UNA_RAW + 1, WORD_MASK,
1484                             f->fs.nat_dip[11] | f->fs.nat_dip[10] << 8 |
1485                             f->fs.nat_dip[9] << 16 | f->fs.nat_dip[8] << 24, 1);
1486
1487                         set_tcb_field(sc, f->tid,
1488                             W_TCB_SND_UNA_RAW + 2, WORD_MASK,
1489                             f->fs.nat_dip[7] | f->fs.nat_dip[6] << 8 |
1490                             f->fs.nat_dip[5] << 16 | f->fs.nat_dip[4] << 24, 1);
1491
1492                         set_tcb_field(sc, f->tid,
1493                             W_TCB_SND_UNA_RAW + 3, WORD_MASK,
1494                             f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
1495                             f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
1496                 } else {
1497                         set_tcb_field(sc, f->tid,
1498                             W_TCB_RX_FRAG3_LEN_RAW, WORD_MASK,
1499                             f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
1500                             f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
1501                 }
1502         }
1503
1504         if (sip) {
1505                 if (f->fs.type) {
1506                         set_tcb_field(sc, f->tid,
1507                             W_TCB_RX_FRAG2_PTR_RAW, WORD_MASK,
1508                             f->fs.nat_sip[15] | f->fs.nat_sip[14] << 8 |
1509                             f->fs.nat_sip[13] << 16 | f->fs.nat_sip[12] << 24, 1);
1510
1511                         set_tcb_field(sc, f->tid,
1512                             W_TCB_RX_FRAG2_PTR_RAW + 1, WORD_MASK,
1513                             f->fs.nat_sip[11] | f->fs.nat_sip[10] << 8 |
1514                             f->fs.nat_sip[9] << 16 | f->fs.nat_sip[8] << 24, 1);
1515
1516                         set_tcb_field(sc, f->tid,
1517                             W_TCB_RX_FRAG2_PTR_RAW + 2, WORD_MASK,
1518                             f->fs.nat_sip[7] | f->fs.nat_sip[6] << 8 |
1519                             f->fs.nat_sip[5] << 16 | f->fs.nat_sip[4] << 24, 1);
1520
1521                         set_tcb_field(sc, f->tid,
1522                             W_TCB_RX_FRAG2_PTR_RAW + 3, WORD_MASK,
1523                             f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
1524                             f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
1525
1526                 } else {
1527                         set_tcb_field(sc, f->tid,
1528                             W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW, WORD_MASK,
1529                             f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
1530                             f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
1531                 }
1532         }
1533
1534         set_tcb_field(sc, f->tid, W_TCB_PDU_HDR_LEN, WORD_MASK,
1535             (dp ? f->fs.nat_dport : 0) | (sp ? f->fs.nat_sport << 16 : 0), 1);
1536 }
1537
1538 /*
1539  * Returns EINPROGRESS to indicate that at least one TCB update was sent and the
1540  * last of the series of updates requested a reply.  The reply informs the
1541  * driver that the filter is fully setup.
1542  */
1543 static int
1544 configure_hashfilter_tcb(struct adapter *sc, struct filter_entry *f)
1545 {
1546         int updated = 0;
1547
1548         MPASS(f->tid < sc->tids.ntids);
1549         MPASS(f->fs.hash);
1550         MPASS(f->pending);
1551         MPASS(f->valid == 0);
1552
1553         if (f->fs.newdmac) {
1554                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECE, 1, 1);
1555                 updated++;
1556         }
1557
1558         if (f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) {
1559                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_RFR, 1, 1);
1560                 updated++;
1561         }
1562
1563         if (f->fs.newsmac) {
1564                 MPASS(f->smt != NULL);
1565                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
1566                 set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
1567                     V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), V_TCB_SMAC_SEL(f->smt->idx),
1568                     1);
1569                 updated++;
1570         }
1571
1572         switch(f->fs.nat_mode) {
1573         case NAT_MODE_NONE:
1574                 break;
1575         case NAT_MODE_DIP:
1576                 set_nat_params(sc, f, true, false, false, false);
1577                 updated++;
1578                 break;
1579         case NAT_MODE_DIP_DP:
1580                 set_nat_params(sc, f, true, false, true, false);
1581                 updated++;
1582                 break;
1583         case NAT_MODE_DIP_DP_SIP:
1584                 set_nat_params(sc, f, true, true, true, false);
1585                 updated++;
1586                 break;
1587         case NAT_MODE_DIP_DP_SP:
1588                 set_nat_params(sc, f, true, false, true, true);
1589                 updated++;
1590                 break;
1591         case NAT_MODE_SIP_SP:
1592                 set_nat_params(sc, f, false, true, false, true);
1593                 updated++;
1594                 break;
1595         case NAT_MODE_DIP_SIP_SP:
1596                 set_nat_params(sc, f, true, true, false, true);
1597                 updated++;
1598                 break;
1599         case NAT_MODE_ALL:
1600                 set_nat_params(sc, f, true, true, true, true);
1601                 updated++;
1602                 break;
1603         default:
1604                 MPASS(0);       /* should have been validated earlier */
1605                 break;
1606
1607         }
1608
1609         if (f->fs.nat_seq_chk) {
1610                 set_tcb_field(sc, f->tid, W_TCB_RCV_NXT,
1611                     V_TCB_RCV_NXT(M_TCB_RCV_NXT),
1612                     V_TCB_RCV_NXT(f->fs.nat_seq_chk), 1);
1613                 updated++;
1614         }
1615
1616         if (is_t5(sc) && f->fs.action == FILTER_DROP) {
1617                 /*
1618                  * Migrating = 1, Non-offload = 0 to get a T5 hashfilter to drop.
1619                  */
1620                 set_tcb_field(sc, f->tid, W_TCB_T_FLAGS, V_TF_NON_OFFLOAD(1) |
1621                     V_TF_MIGRATING(1), V_TF_MIGRATING(1), 1);
1622                 updated++;
1623         }
1624
1625         /*
1626          * Enable switching after all secondary resources (L2T entry, SMT entry,
1627          * etc.) are setup so that any switched packet will use correct
1628          * values.
1629          */
1630         if (f->fs.action == FILTER_SWITCH) {
1631                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECN, 1, 1);
1632                 updated++;
1633         }
1634
1635         if (f->fs.hitcnts || updated > 0) {
1636                 set_tcb_field(sc, f->tid, W_TCB_TIMESTAMP,
1637                     V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) |
1638                     V_TCB_T_RTT_TS_RECENT_AGE(M_TCB_T_RTT_TS_RECENT_AGE),
1639                     V_TCB_TIMESTAMP(0ULL) | V_TCB_T_RTT_TS_RECENT_AGE(0ULL), 0);
1640                 return (EINPROGRESS);
1641         }
1642
1643         return (0);
1644 }