sys/dev/cxgbe/t4_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2018 Chelsio Communications, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/module.h>
  39 #include <sys/bus.h>
  40 #include <sys/lock.h>
  41 #include <sys/mutex.h>
  42 #include <sys/rwlock.h>
  43 #include <sys/socket.h>
  44 #include <sys/sbuf.h>
  45 #include <netinet/in.h>
  46
  47 #include "common/common.h"
  48 #include "common/t4_msg.h"
  49 #include "common/t4_regs.h"
  50 #include "common/t4_regs_values.h"
  51 #include "common/t4_tcb.h"
  52 #include "t4_l2t.h"
  53 #include "t4_smt.h"
  54
  55 struct filter_entry {
  56         uint32_t valid:1;       /* filter allocated and valid */
  57         uint32_t locked:1;      /* filter is administratively locked or busy */
  58         uint32_t pending:1;     /* filter action is pending firmware reply */
  59         int tid;                /* tid of the filter TCB */
  60         struct l2t_entry *l2te; /* L2 table entry for DMAC rewrite */
  61         struct smt_entry *smt;  /* SMT entry for SMAC rewrite */
  62
  63         struct t4_filter_specification fs;
  64 };
  65
  66 static void free_filter_resources(struct filter_entry *);
  67 static int get_hashfilter(struct adapter *, struct t4_filter *);
  68 static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t,
  69     struct l2t_entry *, struct smt_entry *);
  70 static int del_hashfilter(struct adapter *, struct t4_filter *);
  71 static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *);
  72
  73 static int
  74 alloc_hftid_tab(struct tid_info *t, int flags)
  75 {
  76
  77         MPASS(t->ntids > 0);
  78         MPASS(t->hftid_tab == NULL);
  79
  80         t->hftid_tab = malloc(sizeof(*t->hftid_tab) * t->ntids, M_CXGBE,
  81             M_ZERO | flags);
  82         if (t->hftid_tab == NULL)
  83                 return (ENOMEM);
  84         mtx_init(&t->hftid_lock, "T4 hashfilters", 0, MTX_DEF);
  85         cv_init(&t->hftid_cv, "t4hfcv");
  86
  87         return (0);
  88 }
  89
  90 void
  91 free_hftid_tab(struct tid_info *t)
  92 {
  93         int i;
  94
  95         if (t->hftid_tab != NULL) {
  96                 MPASS(t->ntids > 0);
  97                 for (i = 0; t->tids_in_use > 0 && i < t->ntids; i++) {
  98                         if (t->hftid_tab[i] == NULL)
  99                                 continue;
 100                         free(t->hftid_tab[i], M_CXGBE);
 101                         t->tids_in_use--;
 102                 }
 103                 free(t->hftid_tab, M_CXGBE);
 104                 t->hftid_tab = NULL;
 105         }
 106
 107         if (mtx_initialized(&t->hftid_lock)) {
 108                 mtx_destroy(&t->hftid_lock);
 109                 cv_destroy(&t->hftid_cv);
 110         }
 111 }
 112
 113 static void
 114 insert_hftid(struct adapter *sc, int tid, void *ctx, int ntids)
 115 {
 116         struct tid_info *t = &sc->tids;
 117
 118         t->hftid_tab[tid] = ctx;
 119         atomic_add_int(&t->tids_in_use, ntids);
 120 }
 121
 122 static void *
 123 lookup_hftid(struct adapter *sc, int tid)
 124 {
 125         struct tid_info *t = &sc->tids;
 126
 127         return (t->hftid_tab[tid]);
 128 }
 129
 130 static void
 131 remove_hftid(struct adapter *sc, int tid, int ntids)
 132 {
 133         struct tid_info *t = &sc->tids;
 134
 135         t->hftid_tab[tid] = NULL;
 136         atomic_subtract_int(&t->tids_in_use, ntids);
 137 }
 138
 139 static uint32_t
 140 mode_to_fconf(uint32_t mode)
 141 {
 142         uint32_t fconf = 0;
 143
 144         if (mode & T4_FILTER_IP_FRAGMENT)
 145                 fconf |= F_FRAGMENTATION;
 146
 147         if (mode & T4_FILTER_MPS_HIT_TYPE)
 148                 fconf |= F_MPSHITTYPE;
 149
 150         if (mode & T4_FILTER_MAC_IDX)
 151                 fconf |= F_MACMATCH;
 152
 153         if (mode & T4_FILTER_ETH_TYPE)
 154                 fconf |= F_ETHERTYPE;
 155
 156         if (mode & T4_FILTER_IP_PROTO)
 157                 fconf |= F_PROTOCOL;
 158
 159         if (mode & T4_FILTER_IP_TOS)
 160                 fconf |= F_TOS;
 161
 162         if (mode & T4_FILTER_VLAN)
 163                 fconf |= F_VLAN;
 164
 165         if (mode & T4_FILTER_VNIC)
 166                 fconf |= F_VNIC_ID;
 167
 168         if (mode & T4_FILTER_PORT)
 169                 fconf |= F_PORT;
 170
 171         if (mode & T4_FILTER_FCoE)
 172                 fconf |= F_FCOE;
 173
 174         return (fconf);
 175 }
 176
 177 static uint32_t
 178 mode_to_iconf(uint32_t mode)
 179 {
 180
 181         if (mode & T4_FILTER_IC_VNIC)
 182                 return (F_VNIC);
 183         return (0);
 184 }
 185
 186 static int
 187 check_fspec_against_fconf_iconf(struct adapter *sc,
 188     struct t4_filter_specification *fs)
 189 {
 190         struct tp_params *tpp = &sc->params.tp;
 191         uint32_t fconf = 0;
 192
 193         if (fs->val.frag || fs->mask.frag)
 194                 fconf |= F_FRAGMENTATION;
 195
 196         if (fs->val.matchtype || fs->mask.matchtype)
 197                 fconf |= F_MPSHITTYPE;
 198
 199         if (fs->val.macidx || fs->mask.macidx)
 200                 fconf |= F_MACMATCH;
 201
 202         if (fs->val.ethtype || fs->mask.ethtype)
 203                 fconf |= F_ETHERTYPE;
 204
 205         if (fs->val.proto || fs->mask.proto)
 206                 fconf |= F_PROTOCOL;
 207
 208         if (fs->val.tos || fs->mask.tos)
 209                 fconf |= F_TOS;
 210
 211         if (fs->val.vlan_vld || fs->mask.vlan_vld)
 212                 fconf |= F_VLAN;
 213
 214         if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
 215                 fconf |= F_VNIC_ID;
 216                 if (tpp->ingress_config & F_VNIC)
 217                         return (EINVAL);
 218         }
 219
 220         if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
 221                 fconf |= F_VNIC_ID;
 222                 if ((tpp->ingress_config & F_VNIC) == 0)
 223                         return (EINVAL);
 224         }
 225
 226         if (fs->val.iport || fs->mask.iport)
 227                 fconf |= F_PORT;
 228
 229         if (fs->val.fcoe || fs->mask.fcoe)
 230                 fconf |= F_FCOE;
 231
 232         if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map)
 233                 return (E2BIG);
 234
 235         return (0);
 236 }
 237
 238 int
 239 get_filter_mode(struct adapter *sc, uint32_t *mode)
 240 {
 241         struct tp_params *tp = &sc->params.tp;
 242         uint64_t mask;
 243
 244         /* Non-zero incoming value in mode means "hashfilter mode". */
 245         mask = *mode ? tp->hash_filter_mask : UINT64_MAX;
 246
 247         /* Always */
 248         *mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
 249             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
 250
 251 #define CHECK_FIELD(fconf_bit, field_shift, field_mask, mode_bit)  do { \
 252         if (tp->vlan_pri_map & (fconf_bit)) { \
 253                 MPASS(tp->field_shift >= 0); \
 254                 if ((mask >> tp->field_shift & field_mask) == field_mask) \
 255                 *mode |= (mode_bit); \
 256         } \
 257 } while (0)
 258
 259         CHECK_FIELD(F_FRAGMENTATION, frag_shift, M_FT_FRAGMENTATION, T4_FILTER_IP_FRAGMENT);
 260         CHECK_FIELD(F_MPSHITTYPE, matchtype_shift, M_FT_MPSHITTYPE, T4_FILTER_MPS_HIT_TYPE);
 261         CHECK_FIELD(F_MACMATCH, macmatch_shift, M_FT_MACMATCH, T4_FILTER_MAC_IDX);
 262         CHECK_FIELD(F_ETHERTYPE, ethertype_shift, M_FT_ETHERTYPE, T4_FILTER_ETH_TYPE);
 263         CHECK_FIELD(F_PROTOCOL, protocol_shift, M_FT_PROTOCOL, T4_FILTER_IP_PROTO);
 264         CHECK_FIELD(F_TOS, tos_shift, M_FT_TOS, T4_FILTER_IP_TOS);
 265         CHECK_FIELD(F_VLAN, vlan_shift, M_FT_VLAN, T4_FILTER_VLAN);
 266         CHECK_FIELD(F_VNIC_ID, vnic_shift, M_FT_VNIC_ID , T4_FILTER_VNIC);
 267         if (tp->ingress_config & F_VNIC)
 268                 *mode |= T4_FILTER_IC_VNIC;
 269         CHECK_FIELD(F_PORT, port_shift, M_FT_PORT , T4_FILTER_PORT);
 270         CHECK_FIELD(F_FCOE, fcoe_shift, M_FT_FCOE , T4_FILTER_FCoE);
 271 #undef CHECK_FIELD
 272
 273         return (0);
 274 }
 275
 276 int
 277 set_filter_mode(struct adapter *sc, uint32_t mode)
 278 {
 279         struct tp_params *tpp = &sc->params.tp;
 280         uint32_t fconf, iconf;
 281         int rc;
 282
 283         iconf = mode_to_iconf(mode);
 284         if ((iconf ^ tpp->ingress_config) & F_VNIC) {
 285                 /*
 286                  * For now we just complain if A_TP_INGRESS_CONFIG is not
 287                  * already set to the correct value for the requested filter
 288                  * mode.  It's not clear if it's safe to write to this register
 289                  * on the fly.  (And we trust the cached value of the register).
 290                  *
 291                  * check_fspec_against_fconf_iconf and other code that looks at
 292                  * tp->vlan_pri_map and tp->ingress_config needs to be reviewed
 293                  * thorougly before allowing dynamic filter mode changes.
 294                  */
 295                 return (EBUSY);
 296         }
 297
 298         fconf = mode_to_fconf(mode);
 299
 300         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 301             "t4setfm");
 302         if (rc)
 303                 return (rc);
 304
 305         if (sc->tids.ftids_in_use > 0) {
 306                 rc = EBUSY;
 307                 goto done;
 308         }
 309
 310 #ifdef TCP_OFFLOAD
 311         if (uld_active(sc, ULD_TOM)) {
 312                 rc = EBUSY;
 313                 goto done;
 314         }
 315 #endif
 316
 317         rc = -t4_set_filter_mode(sc, fconf, true);
 318 done:
 319         end_synchronized_op(sc, LOCK_HELD);
 320         return (rc);
 321 }
 322
 323 static inline uint64_t
 324 get_filter_hits(struct adapter *sc, uint32_t tid)
 325 {
 326         uint32_t tcb_addr;
 327
 328         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE;
 329
 330         if (is_t4(sc)) {
 331                 uint64_t hits;
 332
 333                 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8);
 334                 return (be64toh(hits));
 335         } else {
 336                 uint32_t hits;
 337
 338                 read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4);
 339                 return (be32toh(hits));
 340         }
 341 }
 342
 343 int
 344 get_filter(struct adapter *sc, struct t4_filter *t)
 345 {
 346         int i, nfilters = sc->tids.nftids;
 347         struct filter_entry *f;
 348
 349         if (t->fs.hash)
 350                 return (get_hashfilter(sc, t));
 351
 352         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
 353             t->idx >= nfilters) {
 354                 t->idx = 0xffffffff;
 355                 return (0);
 356         }
 357
 358         mtx_lock(&sc->tids.ftid_lock);
 359         f = &sc->tids.ftid_tab[t->idx];
 360         for (i = t->idx; i < nfilters; i++, f++) {
 361                 if (f->valid) {
 362                         MPASS(f->tid == sc->tids.ftid_base + i);
 363                         t->idx = i;
 364                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
 365                         t->smtidx = f->smt ? f->smt->idx : 0;
 366                         if (f->fs.hitcnts)
 367                                 t->hits = get_filter_hits(sc, f->tid);
 368                         else
 369                                 t->hits = UINT64_MAX;
 370                         t->fs = f->fs;
 371
 372                         goto done;
 373                 }
 374         }
 375         t->idx = 0xffffffff;
 376 done:
 377         mtx_unlock(&sc->tids.ftid_lock);
 378         return (0);
 379 }
 380
 381 static int
 382 set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te,
 383     struct smt_entry *smt)
 384 {
 385         struct filter_entry *f;
 386         struct fw_filter2_wr *fwr;
 387         u_int vnic_vld, vnic_vld_mask;
 388         struct wrq_cookie cookie;
 389         int i, rc, busy, locked;
 390         const int ntids = t->fs.type ? 4 : 1;
 391
 392         MPASS(!t->fs.hash);
 393         MPASS(t->idx < sc->tids.nftids);
 394         /* Already validated against fconf, iconf */
 395         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
 396         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
 397
 398         f = &sc->tids.ftid_tab[t->idx];
 399         rc = busy = locked = 0;
 400         mtx_lock(&sc->tids.ftid_lock);
 401         for (i = 0; i < ntids; i++) {
 402                 busy += f[i].pending + f[i].valid;
 403                 locked += f[i].locked;
 404         }
 405         if (locked > 0)
 406                 rc = EPERM;
 407         else if (busy > 0)
 408                 rc = EBUSY;
 409         else {
 410                 int len16;
 411
 412                 if (sc->params.filter2_wr_support)
 413                         len16 = howmany(sizeof(struct fw_filter2_wr), 16);
 414                 else
 415                         len16 = howmany(sizeof(struct fw_filter_wr), 16);
 416                 fwr = start_wrq_wr(&sc->sge.mgmtq, len16, &cookie);
 417                 if (__predict_false(fwr == NULL))
 418                         rc = ENOMEM;
 419                 else {
 420                         f->pending = 1;
 421                         sc->tids.ftids_in_use++;
 422                 }
 423         }
 424         mtx_unlock(&sc->tids.ftid_lock);
 425         if (rc != 0) {
 426                 if (l2te)
 427                         t4_l2t_release(l2te);
 428                 if (smt)
 429                         t4_smt_release(smt);
 430                 return (rc);
 431         }
 432
 433         /*
 434          * Can't fail now.  A set-filter WR will definitely be sent.
 435          */
 436
 437         f->tid = sc->tids.ftid_base + t->idx;
 438         f->fs = t->fs;
 439         f->l2te = l2te;
 440         f->smt = smt;
 441
 442         if (t->fs.val.pfvf_vld || t->fs.val.ovlan_vld)
 443                 vnic_vld = 1;
 444         else
 445                 vnic_vld = 0;
 446         if (t->fs.mask.pfvf_vld || t->fs.mask.ovlan_vld)
 447                 vnic_vld_mask = 1;
 448         else
 449                 vnic_vld_mask = 0;
 450
 451         bzero(fwr, sizeof(*fwr));
 452         if (sc->params.filter2_wr_support)
 453                 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER2_WR));
 454         else
 455                 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
 456         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
 457         fwr->tid_to_iq =
 458             htobe32(V_FW_FILTER_WR_TID(f->tid) |
 459                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
 460                 V_FW_FILTER_WR_NOREPLY(0) |
 461                 V_FW_FILTER_WR_IQ(f->fs.iq));
 462         fwr->del_filter_to_l2tix =
 463             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
 464                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
 465                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
 466                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
 467                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
 468                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
 469                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
 470                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
 471                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
 472                     f->fs.newvlan == VLAN_REWRITE) |
 473                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
 474                     f->fs.newvlan == VLAN_REWRITE) |
 475                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
 476                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
 477                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
 478                 V_FW_FILTER_WR_L2TIX(f->l2te ? f->l2te->idx : 0));
 479         fwr->ethtype = htobe16(f->fs.val.ethtype);
 480         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
 481         fwr->frag_to_ovlan_vldm =
 482             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
 483                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
 484                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
 485                 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
 486                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
 487                 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
 488         fwr->smac_sel = 0;
 489         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
 490             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
 491         fwr->maci_to_matchtypem =
 492             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
 493                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
 494                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
 495                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
 496                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
 497                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
 498                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
 499                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
 500         fwr->ptcl = f->fs.val.proto;
 501         fwr->ptclm = f->fs.mask.proto;
 502         fwr->ttyp = f->fs.val.tos;
 503         fwr->ttypm = f->fs.mask.tos;
 504         fwr->ivlan = htobe16(f->fs.val.vlan);
 505         fwr->ivlanm = htobe16(f->fs.mask.vlan);
 506         fwr->ovlan = htobe16(f->fs.val.vnic);
 507         fwr->ovlanm = htobe16(f->fs.mask.vnic);
 508         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
 509         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
 510         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
 511         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
 512         fwr->lp = htobe16(f->fs.val.dport);
 513         fwr->lpm = htobe16(f->fs.mask.dport);
 514         fwr->fp = htobe16(f->fs.val.sport);
 515         fwr->fpm = htobe16(f->fs.mask.sport);
 516         /* sma = 0 tells the fw to use SMAC_SEL for source MAC address */
 517         bzero(fwr->sma, sizeof (fwr->sma));
 518         if (sc->params.filter2_wr_support) {
 519                 fwr->filter_type_swapmac =
 520                     V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac);
 521                 fwr->natmode_to_ulp_type =
 522                     V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ?
 523                         ULP_MODE_TCPDDP : ULP_MODE_NONE) |
 524                     V_FW_FILTER2_WR_NATFLAGCHECK(f->fs.nat_flag_chk) |
 525                     V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode);
 526                 memcpy(fwr->newlip, f->fs.nat_dip, sizeof(fwr->newlip));
 527                 memcpy(fwr->newfip, f->fs.nat_sip, sizeof(fwr->newfip));
 528                 fwr->newlport = htobe16(f->fs.nat_dport);
 529                 fwr->newfport = htobe16(f->fs.nat_sport);
 530                 fwr->natseqcheck = htobe32(f->fs.nat_seq_chk);
 531         }
 532         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
 533
 534         /* Wait for response. */
 535         mtx_lock(&sc->tids.ftid_lock);
 536         for (;;) {
 537                 if (f->pending == 0) {
 538                         rc = f->valid ? 0 : EIO;
 539                         break;
 540                 }
 541                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
 542                         rc = EINPROGRESS;
 543                         break;
 544                 }
 545         }
 546         mtx_unlock(&sc->tids.ftid_lock);
 547         return (rc);
 548 }
 549
 550 static int
 551 hashfilter_ntuple(struct adapter *sc, const struct t4_filter_specification *fs,
 552     uint64_t *ftuple)
 553 {
 554         struct tp_params *tp = &sc->params.tp;
 555         uint64_t fmask;
 556
 557         *ftuple = fmask = 0;
 558
 559         /*
 560          * Initialize each of the fields which we care about which are present
 561          * in the Compressed Filter Tuple.
 562          */
 563         if (tp->vlan_shift >= 0 && fs->mask.vlan) {
 564                 *ftuple |= (F_FT_VLAN_VLD | fs->val.vlan) << tp->vlan_shift;
 565                 fmask |= M_FT_VLAN << tp->vlan_shift;
 566         }
 567
 568         if (tp->port_shift >= 0 && fs->mask.iport) {
 569                 *ftuple |= (uint64_t)fs->val.iport << tp->port_shift;
 570                 fmask |= M_FT_PORT << tp->port_shift;
 571         }
 572
 573         if (tp->protocol_shift >= 0 && fs->mask.proto) {
 574                 *ftuple |= (uint64_t)fs->val.proto << tp->protocol_shift;
 575                 fmask |= M_FT_PROTOCOL << tp->protocol_shift;
 576         }
 577
 578         if (tp->tos_shift >= 0 && fs->mask.tos) {
 579                 *ftuple |= (uint64_t)(fs->val.tos) << tp->tos_shift;
 580                 fmask |= M_FT_TOS << tp->tos_shift;
 581         }
 582
 583         if (tp->vnic_shift >= 0 && fs->mask.vnic) {
 584                 /* F_VNIC in ingress config was already validated. */
 585                 if (tp->ingress_config & F_VNIC)
 586                         MPASS(fs->mask.pfvf_vld);
 587                 else
 588                         MPASS(fs->mask.ovlan_vld);
 589
 590                 *ftuple |= ((1ULL << 16) | fs->val.vnic) << tp->vnic_shift;
 591                 fmask |= M_FT_VNIC_ID << tp->vnic_shift;
 592         }
 593
 594         if (tp->macmatch_shift >= 0 && fs->mask.macidx) {
 595                 *ftuple |= (uint64_t)(fs->val.macidx) << tp->macmatch_shift;
 596                 fmask |= M_FT_MACMATCH << tp->macmatch_shift;
 597         }
 598
 599         if (tp->ethertype_shift >= 0 && fs->mask.ethtype) {
 600                 *ftuple |= (uint64_t)(fs->val.ethtype) << tp->ethertype_shift;
 601                 fmask |= M_FT_ETHERTYPE << tp->ethertype_shift;
 602         }
 603
 604         if (tp->matchtype_shift >= 0 && fs->mask.matchtype) {
 605                 *ftuple |= (uint64_t)(fs->val.matchtype) << tp->matchtype_shift;
 606                 fmask |= M_FT_MPSHITTYPE << tp->matchtype_shift;
 607         }
 608
 609         if (tp->frag_shift >= 0 && fs->mask.frag) {
 610                 *ftuple |= (uint64_t)(fs->val.frag) << tp->frag_shift;
 611                 fmask |= M_FT_FRAGMENTATION << tp->frag_shift;
 612         }
 613
 614         if (tp->fcoe_shift >= 0 && fs->mask.fcoe) {
 615                 *ftuple |= (uint64_t)(fs->val.fcoe) << tp->fcoe_shift;
 616                 fmask |= M_FT_FCOE << tp->fcoe_shift;
 617         }
 618
 619         /* A hashfilter must conform to the filterMask. */
 620         if (fmask != tp->hash_filter_mask)
 621                 return (EINVAL);
 622
 623         return (0);
 624 }
 625
 626 int
 627 set_filter(struct adapter *sc, struct t4_filter *t)
 628 {
 629         struct tid_info *ti = &sc->tids;
 630         struct l2t_entry *l2te;
 631         struct smt_entry *smt;
 632         uint64_t ftuple;
 633         int rc;
 634
 635         /*
 636          * Basic filter checks first.
 637          */
 638
 639         if (t->fs.hash) {
 640                 if (!is_hashfilter(sc) || ti->ntids == 0)
 641                         return (ENOTSUP);
 642                 /* Hardware, not user, selects a tid for hashfilters. */
 643                 if (t->idx != (uint32_t)-1)
 644                         return (EINVAL);
 645                 /* T5 can't count hashfilter hits. */
 646                 if (is_t5(sc) && t->fs.hitcnts)
 647                         return (EINVAL);
 648                 rc = hashfilter_ntuple(sc, &t->fs, &ftuple);
 649                 if (rc != 0)
 650                         return (rc);
 651         } else {
 652                 if (ti->nftids == 0)
 653                         return (ENOTSUP);
 654                 if (t->idx >= ti->nftids)
 655                         return (EINVAL);
 656                 /* IPv6 filter idx must be 4 aligned */
 657                 if (t->fs.type == 1 &&
 658                     ((t->idx & 0x3) || t->idx + 4 >= ti->nftids))
 659                         return (EINVAL);
 660         }
 661
 662         /* T4 doesn't support VLAN tag removal or rewrite, swapmac, and NAT. */
 663         if (is_t4(sc) && t->fs.action == FILTER_SWITCH &&
 664             (t->fs.newvlan == VLAN_REMOVE || t->fs.newvlan == VLAN_REWRITE ||
 665             t->fs.swapmac || t->fs.nat_mode))
 666                 return (ENOTSUP);
 667
 668         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= sc->params.nports)
 669                 return (EINVAL);
 670         if (t->fs.val.iport >= sc->params.nports)
 671                 return (EINVAL);
 672
 673         /* Can't specify an iq if not steering to it */
 674         if (!t->fs.dirsteer && t->fs.iq)
 675                 return (EINVAL);
 676
 677         /* Validate against the global filter mode and ingress config */
 678         rc = check_fspec_against_fconf_iconf(sc, &t->fs);
 679         if (rc != 0)
 680                 return (rc);
 681
 682         /*
 683          * Basic checks passed.  Make sure the queues and tid tables are setup.
 684          */
 685
 686         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
 687         if (rc)
 688                 return (rc);
 689         if (!(sc->flags & FULL_INIT_DONE) &&
 690             ((rc = adapter_full_init(sc)) != 0)) {
 691                 end_synchronized_op(sc, 0);
 692                 return (rc);
 693         }
 694         if (t->fs.hash) {
 695                 if (__predict_false(ti->hftid_tab == NULL)) {
 696                         rc = alloc_hftid_tab(&sc->tids, M_NOWAIT);
 697                         if (rc != 0)
 698                                 goto done;
 699                 }
 700                 if (__predict_false(sc->tids.atid_tab == NULL)) {
 701                         rc = alloc_atid_tab(&sc->tids, M_NOWAIT);
 702                         if (rc != 0)
 703                                 goto done;
 704                 }
 705         } else if (__predict_false(ti->ftid_tab == NULL)) {
 706                 KASSERT(ti->ftids_in_use == 0,
 707                     ("%s: no memory allocated but ftids_in_use > 0", __func__));
 708                 ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids,
 709                     M_CXGBE, M_NOWAIT | M_ZERO);
 710                 if (ti->ftid_tab == NULL) {
 711                         rc = ENOMEM;
 712                         goto done;
 713                 }
 714                 mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
 715                 cv_init(&ti->ftid_cv, "t4fcv");
 716         }
 717 done:
 718         end_synchronized_op(sc, 0);
 719         if (rc != 0)
 720                 return (rc);
 721
 722         /*
 723          * Allocate L2T entry, SMT entry, etc.
 724          */
 725
 726         l2te = NULL;
 727         if (t->fs.newdmac || t->fs.newvlan) {
 728                 /* This filter needs an L2T entry; allocate one. */
 729                 l2te = t4_l2t_alloc_switching(sc->l2t);
 730                 if (__predict_false(l2te == NULL))
 731                         return (EAGAIN);
 732                 rc = t4_l2t_set_switching(sc, l2te, t->fs.vlan, t->fs.eport,
 733                     t->fs.dmac);
 734                 if (rc) {
 735                         t4_l2t_release(l2te);
 736                         return (ENOMEM);
 737                 }
 738         }
 739
 740         smt = NULL;
 741         if (t->fs.newsmac) {
 742                 /* This filter needs an SMT entry; allocate one. */
 743                 smt = t4_smt_alloc_switching(sc->smt, t->fs.smac);
 744                 if (__predict_false(smt == NULL)) {
 745                         if (l2te != NULL)
 746                                 t4_l2t_release(l2te);
 747                         return (EAGAIN);
 748                 }
 749                 rc = t4_smt_set_switching(sc, smt, 0x0, t->fs.smac);
 750                 if (rc) {
 751                         t4_smt_release(smt);
 752                         if (l2te != NULL)
 753                                 t4_l2t_release(l2te);
 754                         return (rc);
 755                 }
 756         }
 757
 758         if (t->fs.hash)
 759                 return (set_hashfilter(sc, t, ftuple, l2te, smt));
 760         else
 761                 return (set_tcamfilter(sc, t, l2te, smt));
 762
 763 }
 764
 765 static int
 766 del_tcamfilter(struct adapter *sc, struct t4_filter *t)
 767 {
 768         struct filter_entry *f;
 769         struct fw_filter_wr *fwr;
 770         struct wrq_cookie cookie;
 771         int rc;
 772
 773         MPASS(sc->tids.ftid_tab != NULL);
 774         MPASS(sc->tids.nftids > 0);
 775
 776         if (t->idx >= sc->tids.nftids)
 777                 return (EINVAL);
 778
 779         mtx_lock(&sc->tids.ftid_lock);
 780         f = &sc->tids.ftid_tab[t->idx];
 781         if (f->locked) {
 782                 rc = EPERM;
 783                 goto done;
 784         }
 785         if (f->pending) {
 786                 rc = EBUSY;
 787                 goto done;
 788         }
 789         if (f->valid == 0) {
 790                 rc = EINVAL;
 791                 goto done;
 792         }
 793         MPASS(f->tid == sc->tids.ftid_base + t->idx);
 794         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
 795         if (fwr == NULL) {
 796                 rc = ENOMEM;
 797                 goto done;
 798         }
 799
 800         bzero(fwr, sizeof (*fwr));
 801         t4_mk_filtdelwr(f->tid, fwr, sc->sge.fwq.abs_id);
 802         f->pending = 1;
 803         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
 804         t->fs = f->fs;  /* extra info for the caller */
 805
 806         for (;;) {
 807                 if (f->pending == 0) {
 808                         rc = f->valid ? EIO : 0;
 809                         break;
 810                 }
 811                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
 812                         rc = EINPROGRESS;
 813                         break;
 814                 }
 815         }
 816 done:
 817         mtx_unlock(&sc->tids.ftid_lock);
 818         return (rc);
 819 }
 820
 821 int
 822 del_filter(struct adapter *sc, struct t4_filter *t)
 823 {
 824
 825         /* No filters possible if not initialized yet. */
 826         if (!(sc->flags & FULL_INIT_DONE))
 827                 return (EINVAL);
 828
 829         /*
 830          * The checks for tid tables ensure that the locks that del_* will reach
 831          * for are initialized.
 832          */
 833         if (t->fs.hash) {
 834                 if (sc->tids.hftid_tab != NULL)
 835                         return (del_hashfilter(sc, t));
 836         } else {
 837                 if (sc->tids.ftid_tab != NULL)
 838                         return (del_tcamfilter(sc, t));
 839         }
 840
 841         return (EINVAL);
 842 }
 843
 844 /*
 845  * Release secondary resources associated with the filter.
 846  */
 847 static void
 848 free_filter_resources(struct filter_entry *f)
 849 {
 850
 851         if (f->l2te) {
 852                 t4_l2t_release(f->l2te);
 853                 f->l2te = NULL;
 854         }
 855         if (f->smt) {
 856                 t4_smt_release(f->smt);
 857                 f->smt = NULL;
 858         }
 859 }
 860
 861 static int
 862 set_tcb_field(struct adapter *sc, u_int tid, uint16_t word, uint64_t mask,
 863     uint64_t val, int no_reply)
 864 {
 865         struct wrq_cookie cookie;
 866         struct cpl_set_tcb_field *req;
 867
 868         req = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*req), 16), &cookie);
 869         if (req == NULL)
 870                 return (ENOMEM);
 871         bzero(req, sizeof(*req));
 872         INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid);
 873         if (no_reply == 0) {
 874                 req->reply_ctrl = htobe16(V_QUEUENO(sc->sge.fwq.abs_id) |
 875                     V_NO_REPLY(0));
 876         } else
 877                 req->reply_ctrl = htobe16(V_NO_REPLY(1));
 878         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(CPL_COOKIE_HASHFILTER));
 879         req->mask = htobe64(mask);
 880         req->val = htobe64(val);
 881         commit_wrq_wr(&sc->sge.mgmtq, req, &cookie);
 882
 883         return (0);
 884 }
 885
 886 /* Set one of the t_flags bits in the TCB. */
 887 static inline int
 888 set_tcb_tflag(struct adapter *sc, int tid, u_int bit_pos, u_int val,
 889     u_int no_reply)
 890 {
 891
 892         return (set_tcb_field(sc, tid,  W_TCB_T_FLAGS, 1ULL << bit_pos,
 893             (uint64_t)val << bit_pos, no_reply));
 894 }
 895
 896 int
 897 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 898 {
 899         struct adapter *sc = iq->adapter;
 900         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
 901         u_int tid = GET_TID(rpl);
 902         u_int rc, cleanup, idx;
 903         struct filter_entry *f;
 904
 905         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
 906             rss->opcode));
 907         MPASS(is_ftid(sc, tid));
 908
 909         cleanup = 0;
 910         idx = tid - sc->tids.ftid_base;
 911         f = &sc->tids.ftid_tab[idx];
 912         rc = G_COOKIE(rpl->cookie);
 913
 914         mtx_lock(&sc->tids.ftid_lock);
 915         KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.",
 916             __func__, rc, idx));
 917         switch(rc) {
 918         case FW_FILTER_WR_FLT_ADDED:
 919                 /* set-filter succeeded */
 920                 f->valid = 1;
 921                 if (f->fs.newsmac) {
 922                         MPASS(f->smt != NULL);
 923                         set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
 924                         set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
 925                             V_TCB_SMAC_SEL(M_TCB_SMAC_SEL),
 926                             V_TCB_SMAC_SEL(f->smt->idx), 1);
 927                         /* XXX: wait for reply to TCB update before !pending */
 928                 }
 929                 break;
 930         case FW_FILTER_WR_FLT_DELETED:
 931                 /* del-filter succeeded */
 932                 MPASS(f->valid == 1);
 933                 f->valid = 0;
 934                 /* Fall through */
 935         case FW_FILTER_WR_SMT_TBL_FULL:
 936                 /* set-filter failed due to lack of SMT space. */
 937                 MPASS(f->valid == 0);
 938                 free_filter_resources(f);
 939                 sc->tids.ftids_in_use--;
 940                 break;
 941         case FW_FILTER_WR_SUCCESS:
 942         case FW_FILTER_WR_EINVAL:
 943         default:
 944                 panic("%s: unexpected reply %d for filter[%d].", __func__, rc,
 945                     idx);
 946         }
 947         f->pending = 0;
 948         cv_broadcast(&sc->tids.ftid_cv);
 949         mtx_unlock(&sc->tids.ftid_lock);
 950
 951         return (0);
 952 }
 953
 954 /*
 955  * This is the reply to the Active Open that created the filter.  Additional TCB
 956  * updates may be required to complete the filter configuration.
 957  */
 958 int
 959 t4_hashfilter_ao_rpl(struct sge_iq *iq, const struct rss_header *rss,
 960     struct mbuf *m)
 961 {
 962         struct adapter *sc = iq->adapter;
 963         const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
 964         u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
 965         u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
 966         struct filter_entry *f = lookup_atid(sc, atid);
 967
 968         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 969
 970         mtx_lock(&sc->tids.hftid_lock);
 971         KASSERT(f->pending, ("%s: hashfilter[%p] isn't pending.", __func__, f));
 972         KASSERT(f->tid == -1, ("%s: hashfilter[%p] has tid %d already.",
 973             __func__, f, f->tid));
 974         if (status == CPL_ERR_NONE) {
 975                 struct filter_entry *f2;
 976
 977                 f->tid = GET_TID(cpl);
 978                 MPASS(f->tid < sc->tids.ntids);
 979                 if (__predict_false((f2 = lookup_hftid(sc, f->tid)) != NULL)) {
 980                         /* XXX: avoid hash collisions in the first place. */
 981                         MPASS(f2->tid == f->tid);
 982                         remove_hftid(sc, f2->tid, f2->fs.type ? 2 : 1);
 983                         free_filter_resources(f2);
 984                         free(f2, M_CXGBE);
 985                 }
 986                 insert_hftid(sc, f->tid, f, f->fs.type ? 2 : 1);
 987                 /*
 988                  * Leave the filter pending until it is fully set up, which will
 989                  * be indicated by the reply to the last TCB update.  No need to
 990                  * unblock the ioctl thread either.
 991                  */
 992                 if (configure_hashfilter_tcb(sc, f) == EINPROGRESS)
 993                         goto done;
 994                 f->valid = 1;
 995                 f->pending = 0;
 996         } else {
 997                 /* provide errno instead of tid to ioctl */
 998                 f->tid = act_open_rpl_status_to_errno(status);
 999                 f->valid = 0;
1000                 if (act_open_has_tid(status))
1001                         release_tid(sc, GET_TID(cpl), &sc->sge.mgmtq);
1002                 free_filter_resources(f);
1003                 if (f->locked == 0)
1004                         free(f, M_CXGBE);
1005         }
1006         cv_broadcast(&sc->tids.hftid_cv);
1007 done:
1008         mtx_unlock(&sc->tids.hftid_lock);
1009
1010         free_atid(sc, atid);
1011         return (0);
1012 }
1013
1014 int
1015 t4_hashfilter_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss,
1016     struct mbuf *m)
1017 {
1018         struct adapter *sc = iq->adapter;
1019         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
1020         u_int tid = GET_TID(rpl);
1021         struct filter_entry *f;
1022
1023         mtx_lock(&sc->tids.hftid_lock);
1024         f = lookup_hftid(sc, tid);
1025         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
1026         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
1027             f, tid));
1028         KASSERT(f->valid == 0, ("%s: hashfilter %p [%u] is valid already.",
1029             __func__, f, tid));
1030         f->pending = 0;
1031         if (rpl->status == 0) {
1032                 f->valid = 1;
1033         } else {
1034                 f->tid = EIO;
1035                 f->valid = 0;
1036                 free_filter_resources(f);
1037                 remove_hftid(sc, tid, f->fs.type ? 2 : 1);
1038                 release_tid(sc, tid, &sc->sge.mgmtq);
1039                 if (f->locked == 0)
1040                         free(f, M_CXGBE);
1041         }
1042         cv_broadcast(&sc->tids.hftid_cv);
1043         mtx_unlock(&sc->tids.hftid_lock);
1044
1045         return (0);
1046 }
1047
1048 int
1049 t4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss,
1050     struct mbuf *m)
1051 {
1052         struct adapter *sc = iq->adapter;
1053         const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1054         unsigned int tid = GET_TID(cpl);
1055         struct filter_entry *f;
1056
1057         mtx_lock(&sc->tids.hftid_lock);
1058         f = lookup_hftid(sc, tid);
1059         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
1060         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
1061             f, tid));
1062         KASSERT(f->valid, ("%s: hashfilter %p [%u] isn't valid.", __func__, f,
1063             tid));
1064         f->pending = 0;
1065         if (cpl->status == 0) {
1066                 f->valid = 0;
1067                 free_filter_resources(f);
1068                 remove_hftid(sc, tid, f->fs.type ? 2 : 1);
1069                 release_tid(sc, tid, &sc->sge.mgmtq);
1070                 if (f->locked == 0)
1071                         free(f, M_CXGBE);
1072         }
1073         cv_broadcast(&sc->tids.hftid_cv);
1074         mtx_unlock(&sc->tids.hftid_lock);
1075
1076         return (0);
1077 }
1078
1079 static int
1080 get_hashfilter(struct adapter *sc, struct t4_filter *t)
1081 {
1082         int i, nfilters = sc->tids.ntids;
1083         struct filter_entry *f;
1084
1085         if (sc->tids.tids_in_use == 0 || sc->tids.hftid_tab == NULL ||
1086             t->idx >= nfilters) {
1087                 t->idx = 0xffffffff;
1088                 return (0);
1089         }
1090
1091         mtx_lock(&sc->tids.hftid_lock);
1092         for (i = t->idx; i < nfilters; i++) {
1093                 f = lookup_hftid(sc, i);
1094                 if (f != NULL && f->valid) {
1095                         t->idx = i;
1096                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
1097                         t->smtidx = f->smt ? f->smt->idx : 0;
1098                         if (f->fs.hitcnts)
1099                                 t->hits = get_filter_hits(sc, t->idx);
1100                         else
1101                                 t->hits = UINT64_MAX;
1102                         t->fs = f->fs;
1103
1104                         goto done;
1105                 }
1106         }
1107         t->idx = 0xffffffff;
1108 done:
1109         mtx_unlock(&sc->tids.hftid_lock);
1110         return (0);
1111 }
1112
1113 static void
1114 mk_act_open_req6(struct adapter *sc, struct filter_entry *f, int atid,
1115     uint64_t ftuple, struct cpl_act_open_req6 *cpl)
1116 {
1117         struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
1118         struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
1119
1120         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1121         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1122         MPASS(atid >= 0);
1123
1124         if (chip_id(sc) == CHELSIO_T5) {
1125                 INIT_TP_WR(cpl5, 0);
1126         } else {
1127                 INIT_TP_WR(cpl6, 0);
1128                 cpl6->rsvd2 = 0;
1129                 cpl6->opt3 = 0;
1130         }
1131
1132         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
1133             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1134             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1135         cpl->local_port = htobe16(f->fs.val.dport);
1136         cpl->peer_port = htobe16(f->fs.val.sport);
1137         cpl->local_ip_hi = *(uint64_t *)(&f->fs.val.dip);
1138         cpl->local_ip_lo = *(((uint64_t *)&f->fs.val.dip) + 1);
1139         cpl->peer_ip_hi = *(uint64_t *)(&f->fs.val.sip);
1140         cpl->peer_ip_lo = *(((uint64_t *)&f->fs.val.sip) + 1);
1141         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1142             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1143             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1144             V_NO_CONG(f->fs.rpttid) |
1145             V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1146             F_TCAM_BYPASS | F_NON_OFFLOAD);
1147
1148         cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1149         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1150             V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1151             V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1152             F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1153             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1154             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1155 }
1156
1157 static void
1158 mk_act_open_req(struct adapter *sc, struct filter_entry *f, int atid,
1159     uint64_t ftuple, struct cpl_act_open_req *cpl)
1160 {
1161         struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
1162         struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
1163
1164         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1165         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1166         MPASS(atid >= 0);
1167
1168         if (chip_id(sc) == CHELSIO_T5) {
1169                 INIT_TP_WR(cpl5, 0);
1170         } else {
1171                 INIT_TP_WR(cpl6, 0);
1172                 cpl6->rsvd2 = 0;
1173                 cpl6->opt3 = 0;
1174         }
1175
1176         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
1177             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1178             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1179         cpl->local_port = htobe16(f->fs.val.dport);
1180         cpl->peer_port = htobe16(f->fs.val.sport);
1181         cpl->local_ip = f->fs.val.dip[0] | f->fs.val.dip[1] << 8 |
1182             f->fs.val.dip[2] << 16 | f->fs.val.dip[3] << 24;
1183         cpl->peer_ip = f->fs.val.sip[0] | f->fs.val.sip[1] << 8 |
1184                 f->fs.val.sip[2] << 16 | f->fs.val.sip[3] << 24;
1185         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1186             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1187             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1188             V_NO_CONG(f->fs.rpttid) |
1189             V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1190             F_TCAM_BYPASS | F_NON_OFFLOAD);
1191
1192         cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1193         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1194             V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1195             V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1196             F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1197             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1198             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1199 }
1200
1201 static int
1202 act_open_cpl_len16(struct adapter *sc, int isipv6)
1203 {
1204         int idx;
1205         static const int sz_table[3][2] = {
1206                 {
1207                         howmany(sizeof (struct cpl_act_open_req), 16),
1208                         howmany(sizeof (struct cpl_act_open_req6), 16)
1209                 },
1210                 {
1211                         howmany(sizeof (struct cpl_t5_act_open_req), 16),
1212                         howmany(sizeof (struct cpl_t5_act_open_req6), 16)
1213                 },
1214                 {
1215                         howmany(sizeof (struct cpl_t6_act_open_req), 16),
1216                         howmany(sizeof (struct cpl_t6_act_open_req6), 16)
1217                 },
1218         };
1219
1220         MPASS(chip_id(sc) >= CHELSIO_T4);
1221         idx = min(chip_id(sc) - CHELSIO_T4, 2);
1222
1223         return (sz_table[idx][!!isipv6]);
1224 }
1225
1226 static int
1227 set_hashfilter(struct adapter *sc, struct t4_filter *t, uint64_t ftuple,
1228     struct l2t_entry *l2te, struct smt_entry *smt)
1229 {
1230         void *wr;
1231         struct wrq_cookie cookie;
1232         struct filter_entry *f;
1233         int rc, atid = -1;
1234
1235         MPASS(t->fs.hash);
1236         /* Already validated against fconf, iconf */
1237         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
1238         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
1239
1240         mtx_lock(&sc->tids.hftid_lock);
1241
1242         /*
1243          * XXX: Check for hash collisions and insert in the hash based lookup
1244          * table so that in-flight hashfilters are also considered when checking
1245          * for collisions.
1246          */
1247
1248         f = malloc(sizeof(*f), M_CXGBE, M_ZERO | M_NOWAIT);
1249         if (__predict_false(f == NULL)) {
1250                 if (l2te)
1251                         t4_l2t_release(l2te);
1252                 if (smt)
1253                         t4_smt_release(smt);
1254                 rc = ENOMEM;
1255                 goto done;
1256         }
1257         f->fs = t->fs;
1258         f->l2te = l2te;
1259         f->smt = smt;
1260
1261         atid = alloc_atid(sc, f);
1262         if (__predict_false(atid) == -1) {
1263                 if (l2te)
1264                         t4_l2t_release(l2te);
1265                 if (smt)
1266                         t4_smt_release(smt);
1267                 free(f, M_CXGBE);
1268                 rc = EAGAIN;
1269                 goto done;
1270         }
1271         MPASS(atid >= 0);
1272
1273         wr = start_wrq_wr(&sc->sge.mgmtq, act_open_cpl_len16(sc, f->fs.type),
1274             &cookie);
1275         if (wr == NULL) {
1276                 free_atid(sc, atid);
1277                 if (l2te)
1278                         t4_l2t_release(l2te);
1279                 if (smt)
1280                         t4_smt_release(smt);
1281                 free(f, M_CXGBE);
1282                 rc = ENOMEM;
1283                 goto done;
1284         }
1285         if (f->fs.type)
1286                 mk_act_open_req6(sc, f, atid, ftuple, wr);
1287         else
1288                 mk_act_open_req(sc, f, atid, ftuple, wr);
1289
1290         f->locked = 1; /* ithread mustn't free f if ioctl is still around. */
1291         f->pending = 1;
1292         f->tid = -1;
1293         commit_wrq_wr(&sc->sge.mgmtq, wr, &cookie);
1294
1295         for (;;) {
1296                 MPASS(f->locked);
1297                 if (f->pending == 0) {
1298                         if (f->valid) {
1299                                 rc = 0;
1300                                 f->locked = 0;
1301                                 t->idx = f->tid;
1302                         } else {
1303                                 rc = f->tid;
1304                                 free(f, M_CXGBE);
1305                         }
1306                         break;
1307                 }
1308                 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1309                         f->locked = 0;
1310                         rc = EINPROGRESS;
1311                         break;
1312                 }
1313         }
1314 done:
1315         mtx_unlock(&sc->tids.hftid_lock);
1316         return (rc);
1317 }
1318
1319 /* SET_TCB_FIELD sent as a ULP command looks like this */
1320 #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \
1321     sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core))
1322
1323 static void *
1324 mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, uint64_t word, uint64_t mask,
1325                 uint64_t val, uint32_t tid, uint32_t qid)
1326 {
1327         struct ulptx_idata *ulpsc;
1328         struct cpl_set_tcb_field_core *req;
1329
1330         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1331         ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
1332
1333         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1334         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1335         ulpsc->len = htobe32(sizeof(*req));
1336
1337         req = (struct cpl_set_tcb_field_core *)(ulpsc + 1);
1338         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1339         req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(qid));
1340         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
1341         req->mask = htobe64(mask);
1342         req->val = htobe64(val);
1343
1344         ulpsc = (struct ulptx_idata *)(req + 1);
1345         if (LEN__SET_TCB_FIELD_ULP % 16) {
1346                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1347                 ulpsc->len = htobe32(0);
1348                 return (ulpsc + 1);
1349         }
1350         return (ulpsc);
1351 }
1352
1353 /* ABORT_REQ sent as a ULP command looks like this */
1354 #define LEN__ABORT_REQ_ULP (sizeof(struct ulp_txpkt) + \
1355         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_req_core))
1356
1357 static void *
1358 mk_abort_req_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1359 {
1360         struct ulptx_idata *ulpsc;
1361         struct cpl_abort_req_core *req;
1362
1363         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1364         ulpmc->len = htobe32(howmany(LEN__ABORT_REQ_ULP, 16));
1365
1366         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1367         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1368         ulpsc->len = htobe32(sizeof(*req));
1369
1370         req = (struct cpl_abort_req_core *)(ulpsc + 1);
1371         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
1372         req->rsvd0 = htonl(0);
1373         req->rsvd1 = 0;
1374         req->cmd = CPL_ABORT_NO_RST;
1375
1376         ulpsc = (struct ulptx_idata *)(req + 1);
1377         if (LEN__ABORT_REQ_ULP % 16) {
1378                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1379                 ulpsc->len = htobe32(0);
1380                 return (ulpsc + 1);
1381         }
1382         return (ulpsc);
1383 }
1384
1385 /* ABORT_RPL sent as a ULP command looks like this */
1386 #define LEN__ABORT_RPL_ULP (sizeof(struct ulp_txpkt) + \
1387         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_rpl_core))
1388
1389 static void *
1390 mk_abort_rpl_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1391 {
1392         struct ulptx_idata *ulpsc;
1393         struct cpl_abort_rpl_core *rpl;
1394
1395         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1396         ulpmc->len = htobe32(howmany(LEN__ABORT_RPL_ULP, 16));
1397
1398         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1399         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1400         ulpsc->len = htobe32(sizeof(*rpl));
1401
1402         rpl = (struct cpl_abort_rpl_core *)(ulpsc + 1);
1403         OPCODE_TID(rpl) = htobe32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
1404         rpl->rsvd0 = htonl(0);
1405         rpl->rsvd1 = 0;
1406         rpl->cmd = CPL_ABORT_NO_RST;
1407
1408         ulpsc = (struct ulptx_idata *)(rpl + 1);
1409         if (LEN__ABORT_RPL_ULP % 16) {
1410                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1411                 ulpsc->len = htobe32(0);
1412                 return (ulpsc + 1);
1413         }
1414         return (ulpsc);
1415 }
1416
1417 static inline int
1418 del_hashfilter_wrlen(void)
1419 {
1420
1421         return (sizeof(struct work_request_hdr) +
1422             roundup2(LEN__SET_TCB_FIELD_ULP, 16) +
1423             roundup2(LEN__ABORT_REQ_ULP, 16) +
1424             roundup2(LEN__ABORT_RPL_ULP, 16));
1425 }
1426
1427 static void
1428 mk_del_hashfilter_wr(int tid, struct work_request_hdr *wrh, int wrlen, int qid)
1429 {
1430         struct ulp_txpkt *ulpmc;
1431
1432         INIT_ULPTX_WRH(wrh, wrlen, 0, 0);
1433         ulpmc = (struct ulp_txpkt *)(wrh + 1);
1434         ulpmc = mk_set_tcb_field_ulp(ulpmc, W_TCB_RSS_INFO,
1435             V_TCB_RSS_INFO(M_TCB_RSS_INFO), V_TCB_RSS_INFO(qid), tid, 0);
1436         ulpmc = mk_abort_req_ulp(ulpmc, tid);
1437         ulpmc = mk_abort_rpl_ulp(ulpmc, tid);
1438 }
1439
1440 static int
1441 del_hashfilter(struct adapter *sc, struct t4_filter *t)
1442 {
1443         void *wr;
1444         struct filter_entry *f;
1445         struct wrq_cookie cookie;
1446         int rc;
1447         const int wrlen = del_hashfilter_wrlen();
1448
1449         MPASS(sc->tids.hftid_tab != NULL);
1450         MPASS(sc->tids.ntids > 0);
1451
1452         if (t->idx >= sc->tids.ntids)
1453                 return (EINVAL);
1454
1455         mtx_lock(&sc->tids.hftid_lock);
1456         f = lookup_hftid(sc, t->idx);
1457         if (f == NULL || f->valid == 0) {
1458                 rc = EINVAL;
1459                 goto done;
1460         }
1461         MPASS(f->tid == t->idx);
1462         if (f->locked) {
1463                 rc = EPERM;
1464                 goto done;
1465         }
1466         if (f->pending) {
1467                 rc = EBUSY;
1468                 goto done;
1469         }
1470         wr = start_wrq_wr(&sc->sge.mgmtq, howmany(wrlen, 16), &cookie);
1471         if (wr == NULL) {
1472                 rc = ENOMEM;
1473                 goto done;
1474         }
1475
1476         mk_del_hashfilter_wr(t->idx, wr, wrlen, sc->sge.fwq.abs_id);
1477         f->locked = 1;
1478         f->pending = 1;
1479         commit_wrq_wr(&sc->sge.mgmtq, wr, &cookie);
1480         t->fs = f->fs;  /* extra info for the caller */
1481
1482         for (;;) {
1483                 MPASS(f->locked);
1484                 if (f->pending == 0) {
1485                         if (f->valid) {
1486                                 f->locked = 0;
1487                                 rc = EIO;
1488                         } else {
1489                                 rc = 0;
1490                                 free(f, M_CXGBE);
1491                         }
1492                         break;
1493                 }
1494                 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1495                         f->locked = 0;
1496                         rc = EINPROGRESS;
1497                         break;
1498                 }
1499         }
1500 done:
1501         mtx_unlock(&sc->tids.hftid_lock);
1502         return (rc);
1503 }
1504
1505 #define WORD_MASK       0xffffffff
1506 static void
1507 set_nat_params(struct adapter *sc, struct filter_entry *f, const bool dip,
1508     const bool sip, const bool dp, const bool sp)
1509 {
1510
1511         if (dip) {
1512                 if (f->fs.type) {
1513                         set_tcb_field(sc, f->tid, W_TCB_SND_UNA_RAW, WORD_MASK,
1514                             f->fs.nat_dip[15] | f->fs.nat_dip[14] << 8 |
1515                             f->fs.nat_dip[13] << 16 | f->fs.nat_dip[12] << 24, 1);
1516
1517                         set_tcb_field(sc, f->tid,
1518                             W_TCB_SND_UNA_RAW + 1, WORD_MASK,
1519                             f->fs.nat_dip[11] | f->fs.nat_dip[10] << 8 |
1520                             f->fs.nat_dip[9] << 16 | f->fs.nat_dip[8] << 24, 1);
1521
1522                         set_tcb_field(sc, f->tid,
1523                             W_TCB_SND_UNA_RAW + 2, WORD_MASK,
1524                             f->fs.nat_dip[7] | f->fs.nat_dip[6] << 8 |
1525                             f->fs.nat_dip[5] << 16 | f->fs.nat_dip[4] << 24, 1);
1526
1527                         set_tcb_field(sc, f->tid,
1528                             W_TCB_SND_UNA_RAW + 3, WORD_MASK,
1529                             f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
1530                             f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
1531                 } else {
1532                         set_tcb_field(sc, f->tid,
1533                             W_TCB_RX_FRAG3_LEN_RAW, WORD_MASK,
1534                             f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
1535                             f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
1536                 }
1537         }
1538
1539         if (sip) {
1540                 if (f->fs.type) {
1541                         set_tcb_field(sc, f->tid,
1542                             W_TCB_RX_FRAG2_PTR_RAW, WORD_MASK,
1543                             f->fs.nat_sip[15] | f->fs.nat_sip[14] << 8 |
1544                             f->fs.nat_sip[13] << 16 | f->fs.nat_sip[12] << 24, 1);
1545
1546                         set_tcb_field(sc, f->tid,
1547                             W_TCB_RX_FRAG2_PTR_RAW + 1, WORD_MASK,
1548                             f->fs.nat_sip[11] | f->fs.nat_sip[10] << 8 |
1549                             f->fs.nat_sip[9] << 16 | f->fs.nat_sip[8] << 24, 1);
1550
1551                         set_tcb_field(sc, f->tid,
1552                             W_TCB_RX_FRAG2_PTR_RAW + 2, WORD_MASK,
1553                             f->fs.nat_sip[7] | f->fs.nat_sip[6] << 8 |
1554                             f->fs.nat_sip[5] << 16 | f->fs.nat_sip[4] << 24, 1);
1555
1556                         set_tcb_field(sc, f->tid,
1557                             W_TCB_RX_FRAG2_PTR_RAW + 3, WORD_MASK,
1558                             f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
1559                             f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
1560
1561                 } else {
1562                         set_tcb_field(sc, f->tid,
1563                             W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW, WORD_MASK,
1564                             f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
1565                             f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
1566                 }
1567         }
1568
1569         set_tcb_field(sc, f->tid, W_TCB_PDU_HDR_LEN, WORD_MASK,
1570             (dp ? f->fs.nat_dport : 0) | (sp ? f->fs.nat_sport << 16 : 0), 1);
1571 }
1572
1573 /*
1574  * Returns EINPROGRESS to indicate that at least one TCB update was sent and the
1575  * last of the series of updates requested a reply.  The reply informs the
1576  * driver that the filter is fully setup.
1577  */
1578 static int
1579 configure_hashfilter_tcb(struct adapter *sc, struct filter_entry *f)
1580 {
1581         int updated = 0;
1582
1583         MPASS(f->tid < sc->tids.ntids);
1584         MPASS(f->fs.hash);
1585         MPASS(f->pending);
1586         MPASS(f->valid == 0);
1587
1588         if (f->fs.newdmac) {
1589                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECE, 1, 1);
1590                 updated++;
1591         }
1592
1593         if (f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) {
1594                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_RFR, 1, 1);
1595                 updated++;
1596         }
1597
1598         if (f->fs.newsmac) {
1599                 MPASS(f->smt != NULL);
1600                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
1601                 set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
1602                     V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), V_TCB_SMAC_SEL(f->smt->idx),
1603                     1);
1604                 updated++;
1605         }
1606
1607         switch(f->fs.nat_mode) {
1608         case NAT_MODE_NONE:
1609                 break;
1610         case NAT_MODE_DIP:
1611                 set_nat_params(sc, f, true, false, false, false);
1612                 updated++;
1613                 break;
1614         case NAT_MODE_DIP_DP:
1615                 set_nat_params(sc, f, true, false, true, false);
1616                 updated++;
1617                 break;
1618         case NAT_MODE_DIP_DP_SIP:
1619                 set_nat_params(sc, f, true, true, true, false);
1620                 updated++;
1621                 break;
1622         case NAT_MODE_DIP_DP_SP:
1623                 set_nat_params(sc, f, true, false, true, true);
1624                 updated++;
1625                 break;
1626         case NAT_MODE_SIP_SP:
1627                 set_nat_params(sc, f, false, true, false, true);
1628                 updated++;
1629                 break;
1630         case NAT_MODE_DIP_SIP_SP:
1631                 set_nat_params(sc, f, true, true, false, true);
1632                 updated++;
1633                 break;
1634         case NAT_MODE_ALL:
1635                 set_nat_params(sc, f, true, true, true, true);
1636                 updated++;
1637                 break;
1638         default:
1639                 MPASS(0);       /* should have been validated earlier */
1640                 break;
1641
1642         }
1643
1644         if (f->fs.nat_seq_chk) {
1645                 set_tcb_field(sc, f->tid, W_TCB_RCV_NXT,
1646                     V_TCB_RCV_NXT(M_TCB_RCV_NXT),
1647                     V_TCB_RCV_NXT(f->fs.nat_seq_chk), 1);
1648                 updated++;
1649         }
1650
1651         if (is_t5(sc) && f->fs.action == FILTER_DROP) {
1652                 /*
1653                  * Migrating = 1, Non-offload = 0 to get a T5 hashfilter to drop.
1654                  */
1655                 set_tcb_field(sc, f->tid, W_TCB_T_FLAGS, V_TF_NON_OFFLOAD(1) |
1656                     V_TF_MIGRATING(1), V_TF_MIGRATING(1), 1);
1657                 updated++;
1658         }
1659
1660         /*
1661          * Enable switching after all secondary resources (L2T entry, SMT entry,
1662          * etc.) are setup so that any switched packet will use correct
1663          * values.
1664          */
1665         if (f->fs.action == FILTER_SWITCH) {
1666                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECN, 1, 1);
1667                 updated++;
1668         }
1669
1670         if (f->fs.hitcnts || updated > 0) {
1671                 set_tcb_field(sc, f->tid, W_TCB_TIMESTAMP,
1672                     V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) |
1673                     V_TCB_T_RTT_TS_RECENT_AGE(M_TCB_T_RTT_TS_RECENT_AGE),
1674                     V_TCB_TIMESTAMP(0ULL) | V_TCB_T_RTT_TS_RECENT_AGE(0ULL), 0);
1675                 return (EINPROGRESS);
1676         }
1677
1678         return (0);
1679 }