sys/dev/cxgbe/t4_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2018 Chelsio Communications, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/module.h>
  39 #include <sys/bus.h>
  40 #include <sys/lock.h>
  41 #include <sys/mutex.h>
  42 #include <sys/rwlock.h>
  43 #include <sys/socket.h>
  44 #include <sys/sbuf.h>
  45 #include <netinet/in.h>
  46
  47 #include "common/common.h"
  48 #include "common/t4_msg.h"
  49 #include "common/t4_regs.h"
  50 #include "common/t4_regs_values.h"
  51 #include "common/t4_tcb.h"
  52 #include "t4_l2t.h"
  53
  54 struct filter_entry {
  55         uint32_t valid:1;       /* filter allocated and valid */
  56         uint32_t locked:1;      /* filter is administratively locked or busy */
  57         uint32_t pending:1;     /* filter action is pending firmware reply */
  58         uint32_t smtidx:8;      /* Source MAC Table index for smac */
  59         int tid;                /* tid of the filter TCB */
  60         struct l2t_entry *l2te; /* L2 table entry for DMAC rewrite */
  61
  62         struct t4_filter_specification fs;
  63 };
  64
  65 static void free_filter_resources(struct filter_entry *);
  66 static int get_hashfilter(struct adapter *, struct t4_filter *);
  67 static int set_hashfilter(struct adapter *, struct t4_filter *,
  68     struct l2t_entry *);
  69 static int del_hashfilter(struct adapter *, struct t4_filter *);
  70 static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *);
  71
  72 static void
  73 insert_hftid(struct adapter *sc, int tid, void *ctx, int ntids)
  74 {
  75         struct tid_info *t = &sc->tids;
  76
  77         t->hftid_tab[tid] = ctx;
  78         atomic_add_int(&t->tids_in_use, ntids);
  79 }
  80
  81 static void *
  82 lookup_hftid(struct adapter *sc, int tid)
  83 {
  84         struct tid_info *t = &sc->tids;
  85
  86         return (t->hftid_tab[tid]);
  87 }
  88
  89 static void
  90 remove_hftid(struct adapter *sc, int tid, int ntids)
  91 {
  92         struct tid_info *t = &sc->tids;
  93
  94         t->hftid_tab[tid] = NULL;
  95         atomic_subtract_int(&t->tids_in_use, ntids);
  96 }
  97
  98 static uint32_t
  99 fconf_iconf_to_mode(uint32_t fconf, uint32_t iconf)
 100 {
 101         uint32_t mode;
 102
 103         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
 104             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
 105
 106         if (fconf & F_FRAGMENTATION)
 107                 mode |= T4_FILTER_IP_FRAGMENT;
 108
 109         if (fconf & F_MPSHITTYPE)
 110                 mode |= T4_FILTER_MPS_HIT_TYPE;
 111
 112         if (fconf & F_MACMATCH)
 113                 mode |= T4_FILTER_MAC_IDX;
 114
 115         if (fconf & F_ETHERTYPE)
 116                 mode |= T4_FILTER_ETH_TYPE;
 117
 118         if (fconf & F_PROTOCOL)
 119                 mode |= T4_FILTER_IP_PROTO;
 120
 121         if (fconf & F_TOS)
 122                 mode |= T4_FILTER_IP_TOS;
 123
 124         if (fconf & F_VLAN)
 125                 mode |= T4_FILTER_VLAN;
 126
 127         if (fconf & F_VNIC_ID) {
 128                 mode |= T4_FILTER_VNIC;
 129                 if (iconf & F_VNIC)
 130                         mode |= T4_FILTER_IC_VNIC;
 131         }
 132
 133         if (fconf & F_PORT)
 134                 mode |= T4_FILTER_PORT;
 135
 136         if (fconf & F_FCOE)
 137                 mode |= T4_FILTER_FCoE;
 138
 139         return (mode);
 140 }
 141
 142 static uint32_t
 143 mode_to_fconf(uint32_t mode)
 144 {
 145         uint32_t fconf = 0;
 146
 147         if (mode & T4_FILTER_IP_FRAGMENT)
 148                 fconf |= F_FRAGMENTATION;
 149
 150         if (mode & T4_FILTER_MPS_HIT_TYPE)
 151                 fconf |= F_MPSHITTYPE;
 152
 153         if (mode & T4_FILTER_MAC_IDX)
 154                 fconf |= F_MACMATCH;
 155
 156         if (mode & T4_FILTER_ETH_TYPE)
 157                 fconf |= F_ETHERTYPE;
 158
 159         if (mode & T4_FILTER_IP_PROTO)
 160                 fconf |= F_PROTOCOL;
 161
 162         if (mode & T4_FILTER_IP_TOS)
 163                 fconf |= F_TOS;
 164
 165         if (mode & T4_FILTER_VLAN)
 166                 fconf |= F_VLAN;
 167
 168         if (mode & T4_FILTER_VNIC)
 169                 fconf |= F_VNIC_ID;
 170
 171         if (mode & T4_FILTER_PORT)
 172                 fconf |= F_PORT;
 173
 174         if (mode & T4_FILTER_FCoE)
 175                 fconf |= F_FCOE;
 176
 177         return (fconf);
 178 }
 179
 180 static uint32_t
 181 mode_to_iconf(uint32_t mode)
 182 {
 183
 184         if (mode & T4_FILTER_IC_VNIC)
 185                 return (F_VNIC);
 186         return (0);
 187 }
 188
 189 static int check_fspec_against_fconf_iconf(struct adapter *sc,
 190     struct t4_filter_specification *fs)
 191 {
 192         struct tp_params *tpp = &sc->params.tp;
 193         uint32_t fconf = 0;
 194
 195         if (fs->val.frag || fs->mask.frag)
 196                 fconf |= F_FRAGMENTATION;
 197
 198         if (fs->val.matchtype || fs->mask.matchtype)
 199                 fconf |= F_MPSHITTYPE;
 200
 201         if (fs->val.macidx || fs->mask.macidx)
 202                 fconf |= F_MACMATCH;
 203
 204         if (fs->val.ethtype || fs->mask.ethtype)
 205                 fconf |= F_ETHERTYPE;
 206
 207         if (fs->val.proto || fs->mask.proto)
 208                 fconf |= F_PROTOCOL;
 209
 210         if (fs->val.tos || fs->mask.tos)
 211                 fconf |= F_TOS;
 212
 213         if (fs->val.vlan_vld || fs->mask.vlan_vld)
 214                 fconf |= F_VLAN;
 215
 216         if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
 217                 fconf |= F_VNIC_ID;
 218                 if (tpp->ingress_config & F_VNIC)
 219                         return (EINVAL);
 220         }
 221
 222         if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
 223                 fconf |= F_VNIC_ID;
 224                 if ((tpp->ingress_config & F_VNIC) == 0)
 225                         return (EINVAL);
 226         }
 227
 228         if (fs->val.iport || fs->mask.iport)
 229                 fconf |= F_PORT;
 230
 231         if (fs->val.fcoe || fs->mask.fcoe)
 232                 fconf |= F_FCOE;
 233
 234         if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map)
 235                 return (E2BIG);
 236
 237         return (0);
 238 }
 239
 240 int
 241 get_filter_mode(struct adapter *sc, uint32_t *mode)
 242 {
 243         struct tp_params *tpp = &sc->params.tp;
 244
 245         /*
 246          * We trust the cached values of the relevant TP registers.  This means
 247          * things work reliably only if writes to those registers are always via
 248          * t4_set_filter_mode.
 249          */
 250         *mode = fconf_iconf_to_mode(tpp->vlan_pri_map, tpp->ingress_config);
 251
 252         return (0);
 253 }
 254
 255 int
 256 set_filter_mode(struct adapter *sc, uint32_t mode)
 257 {
 258         struct tp_params *tpp = &sc->params.tp;
 259         uint32_t fconf, iconf;
 260         int rc;
 261
 262         iconf = mode_to_iconf(mode);
 263         if ((iconf ^ tpp->ingress_config) & F_VNIC) {
 264                 /*
 265                  * For now we just complain if A_TP_INGRESS_CONFIG is not
 266                  * already set to the correct value for the requested filter
 267                  * mode.  It's not clear if it's safe to write to this register
 268                  * on the fly.  (And we trust the cached value of the register).
 269                  *
 270                  * check_fspec_against_fconf_iconf and other code that looks at
 271                  * tp->vlan_pri_map and tp->ingress_config needs to be reviewed
 272                  * thorougly before allowing dynamic filter mode changes.
 273                  */
 274                 return (EBUSY);
 275         }
 276
 277         fconf = mode_to_fconf(mode);
 278
 279         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
 280             "t4setfm");
 281         if (rc)
 282                 return (rc);
 283
 284         if (sc->tids.ftids_in_use > 0) {
 285                 rc = EBUSY;
 286                 goto done;
 287         }
 288
 289 #ifdef TCP_OFFLOAD
 290         if (uld_active(sc, ULD_TOM)) {
 291                 rc = EBUSY;
 292                 goto done;
 293         }
 294 #endif
 295
 296         rc = -t4_set_filter_mode(sc, fconf, true);
 297 done:
 298         end_synchronized_op(sc, LOCK_HELD);
 299         return (rc);
 300 }
 301
 302 static inline uint64_t
 303 get_filter_hits(struct adapter *sc, uint32_t tid)
 304 {
 305         uint32_t tcb_addr;
 306
 307         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE;
 308
 309         if (is_t4(sc)) {
 310                 uint64_t hits;
 311
 312                 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8);
 313                 return (be64toh(hits));
 314         } else {
 315                 uint32_t hits;
 316
 317                 read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4);
 318                 return (be32toh(hits));
 319         }
 320 }
 321
 322 int
 323 get_filter(struct adapter *sc, struct t4_filter *t)
 324 {
 325         int i, nfilters = sc->tids.nftids;
 326         struct filter_entry *f;
 327
 328         if (t->fs.hash)
 329                 return (get_hashfilter(sc, t));
 330
 331         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
 332             t->idx >= nfilters) {
 333                 t->idx = 0xffffffff;
 334                 return (0);
 335         }
 336
 337         mtx_lock(&sc->tids.ftid_lock);
 338         f = &sc->tids.ftid_tab[t->idx];
 339         MPASS(f->tid == sc->tids.ftid_base + t->idx);
 340         for (i = t->idx; i < nfilters; i++, f++) {
 341                 if (f->valid) {
 342                         t->idx = i;
 343                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
 344                         t->smtidx = f->smtidx;
 345                         if (f->fs.hitcnts)
 346                                 t->hits = get_filter_hits(sc, f->tid);
 347                         else
 348                                 t->hits = UINT64_MAX;
 349                         t->fs = f->fs;
 350
 351                         goto done;
 352                 }
 353         }
 354         t->idx = 0xffffffff;
 355 done:
 356         mtx_unlock(&sc->tids.ftid_lock);
 357         return (0);
 358 }
 359
 360 static int
 361 set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te)
 362 {
 363         struct filter_entry *f;
 364         struct fw_filter_wr *fwr;
 365         u_int vnic_vld, vnic_vld_mask;
 366         struct wrq_cookie cookie;
 367         int i, rc, busy, locked;
 368         const int ntids = t->fs.type ? 4 : 1;
 369
 370         MPASS(!t->fs.hash);
 371         MPASS(t->idx < sc->tids.nftids);
 372         /* Already validated against fconf, iconf */
 373         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
 374         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
 375
 376         f = &sc->tids.ftid_tab[t->idx];
 377         rc = busy = locked = 0;
 378         mtx_lock(&sc->tids.ftid_lock);
 379         for (i = 0; i < ntids; i++) {
 380                 busy += f[i].pending + f[i].valid;
 381                 locked += f[i].locked;
 382         }
 383         if (locked > 0)
 384                 rc = EPERM;
 385         else if (busy > 0)
 386                 rc = EBUSY;
 387         else {
 388                 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16),
 389                     &cookie);
 390                 if (__predict_false(fwr == NULL))
 391                         rc = ENOMEM;
 392                 else {
 393                         f->pending = 1;
 394                         sc->tids.ftids_in_use++;
 395                 }
 396         }
 397         mtx_unlock(&sc->tids.ftid_lock);
 398         if (rc != 0) {
 399                 if (l2te)
 400                         t4_l2t_release(l2te);
 401                 return (rc);
 402         }
 403
 404         /*
 405          * Can't fail now.  A set-filter WR will definitely be sent.
 406          */
 407
 408         f->tid = sc->tids.ftid_base + t->idx;
 409         f->fs = t->fs;
 410         f->l2te = l2te;
 411
 412         if (t->fs.val.pfvf_vld || t->fs.val.ovlan_vld)
 413                 vnic_vld = 1;
 414         else
 415                 vnic_vld = 0;
 416         if (t->fs.mask.pfvf_vld || t->fs.mask.ovlan_vld)
 417                 vnic_vld_mask = 1;
 418         else
 419                 vnic_vld_mask = 0;
 420
 421         bzero(fwr, sizeof(*fwr));
 422         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
 423         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
 424         fwr->tid_to_iq =
 425             htobe32(V_FW_FILTER_WR_TID(f->tid) |
 426                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
 427                 V_FW_FILTER_WR_NOREPLY(0) |
 428                 V_FW_FILTER_WR_IQ(f->fs.iq));
 429         fwr->del_filter_to_l2tix =
 430             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
 431                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
 432                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
 433                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
 434                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
 435                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
 436                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
 437                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
 438                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
 439                     f->fs.newvlan == VLAN_REWRITE) |
 440                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
 441                     f->fs.newvlan == VLAN_REWRITE) |
 442                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
 443                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
 444                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
 445                 V_FW_FILTER_WR_L2TIX(f->l2te ? f->l2te->idx : 0));
 446         fwr->ethtype = htobe16(f->fs.val.ethtype);
 447         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
 448         fwr->frag_to_ovlan_vldm =
 449             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
 450                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
 451                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
 452                 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
 453                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
 454                 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
 455         fwr->smac_sel = 0;
 456         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
 457             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
 458         fwr->maci_to_matchtypem =
 459             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
 460                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
 461                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
 462                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
 463                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
 464                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
 465                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
 466                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
 467         fwr->ptcl = f->fs.val.proto;
 468         fwr->ptclm = f->fs.mask.proto;
 469         fwr->ttyp = f->fs.val.tos;
 470         fwr->ttypm = f->fs.mask.tos;
 471         fwr->ivlan = htobe16(f->fs.val.vlan);
 472         fwr->ivlanm = htobe16(f->fs.mask.vlan);
 473         fwr->ovlan = htobe16(f->fs.val.vnic);
 474         fwr->ovlanm = htobe16(f->fs.mask.vnic);
 475         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
 476         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
 477         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
 478         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
 479         fwr->lp = htobe16(f->fs.val.dport);
 480         fwr->lpm = htobe16(f->fs.mask.dport);
 481         fwr->fp = htobe16(f->fs.val.sport);
 482         fwr->fpm = htobe16(f->fs.mask.sport);
 483         if (f->fs.newsmac) {
 484                 /* XXX: need to use SMT idx instead */
 485                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
 486         }
 487         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
 488
 489         /* Wait for response. */
 490         mtx_lock(&sc->tids.ftid_lock);
 491         for (;;) {
 492                 if (f->pending == 0) {
 493                         rc = f->valid ? 0 : EIO;
 494                         break;
 495                 }
 496                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
 497                         rc = EINPROGRESS;
 498                         break;
 499                 }
 500         }
 501         mtx_unlock(&sc->tids.ftid_lock);
 502         return (rc);
 503 }
 504
 505 int
 506 set_filter(struct adapter *sc, struct t4_filter *t)
 507 {
 508         struct tid_info *ti = &sc->tids;
 509         struct l2t_entry *l2te;
 510         int rc;
 511
 512         /*
 513          * Basic filter checks first.
 514          */
 515
 516         if (t->fs.hash) {
 517                 if (!is_hashfilter(sc) || ti->ntids == 0)
 518                         return (ENOTSUP);
 519                 if (t->idx != (uint32_t)-1)
 520                         return (EINVAL);        /* hw, not user picks the idx */
 521         } else {
 522                 if (ti->nftids == 0)
 523                         return (ENOTSUP);
 524                 if (t->idx >= ti->nftids)
 525                         return (EINVAL);
 526                 /* IPv6 filter idx must be 4 aligned */
 527                 if (t->fs.type == 1 &&
 528                     ((t->idx & 0x3) || t->idx + 4 >= ti->nftids))
 529                         return (EINVAL);
 530         }
 531
 532         /* T4 doesn't support removing VLAN Tags for loop back filters. */
 533         if (is_t4(sc) && t->fs.action == FILTER_SWITCH &&
 534             (t->fs.newvlan == VLAN_REMOVE || t->fs.newvlan == VLAN_REWRITE))
 535                 return (ENOTSUP);
 536
 537         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= sc->params.nports)
 538                 return (EINVAL);
 539         if (t->fs.val.iport >= sc->params.nports)
 540                 return (EINVAL);
 541
 542         /* Can't specify an iq if not steering to it */
 543         if (!t->fs.dirsteer && t->fs.iq)
 544                 return (EINVAL);
 545
 546         /* Validate against the global filter mode and ingress config */
 547         rc = check_fspec_against_fconf_iconf(sc, &t->fs);
 548         if (rc != 0)
 549                 return (rc);
 550
 551         /*
 552          * Basic checks passed.  Make sure the queues and tid tables are setup.
 553          */
 554
 555         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
 556         if (rc)
 557                 return (rc);
 558         if (!(sc->flags & FULL_INIT_DONE) &&
 559             ((rc = adapter_full_init(sc)) != 0)) {
 560                 end_synchronized_op(sc, 0);
 561                 return (rc);
 562         }
 563         if (t->fs.hash) {
 564                 if (__predict_false(ti->hftid_tab == NULL)) {
 565                         ti->hftid_tab = malloc(sizeof(*ti->hftid_tab) * ti->ntids,
 566                             M_CXGBE, M_NOWAIT | M_ZERO);
 567                         if (ti->hftid_tab == NULL) {
 568                                 rc = ENOMEM;
 569                                 goto done;
 570                         }
 571                         mtx_init(&ti->hftid_lock, "T4 hashfilters", 0, MTX_DEF);
 572                         cv_init(&ti->hftid_cv, "t4hfcv");
 573                 }
 574                 if (__predict_false(sc->tids.atid_tab == NULL)) {
 575                         rc = alloc_atid_tab(&sc->tids, M_NOWAIT);
 576                         if (rc != 0)
 577                                 goto done;
 578                 }
 579         } else if (__predict_false(ti->ftid_tab == NULL)) {
 580                 KASSERT(ti->ftids_in_use == 0,
 581                     ("%s: no memory allocated but ftids_in_use > 0", __func__));
 582                 ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids,
 583                     M_CXGBE, M_NOWAIT | M_ZERO);
 584                 if (ti->ftid_tab == NULL) {
 585                         rc = ENOMEM;
 586                         goto done;
 587                 }
 588                 mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
 589                 cv_init(&ti->ftid_cv, "t4fcv");
 590         }
 591 done:
 592         end_synchronized_op(sc, 0);
 593         if (rc != 0)
 594                 return (rc);
 595
 596         /*
 597          * Allocate L2T entry, SMT entry, etc.
 598          */
 599
 600         l2te = NULL;
 601         if (t->fs.newdmac || t->fs.newvlan) {
 602                 /* This filter needs an L2T entry; allocate one. */
 603                 l2te = t4_l2t_alloc_switching(sc->l2t);
 604                 if (__predict_false(l2te == NULL))
 605                         return (EAGAIN);
 606                 if (t4_l2t_set_switching(sc, l2te, t->fs.vlan, t->fs.eport,
 607                     t->fs.dmac)) {
 608                         t4_l2t_release(l2te);
 609                         return (ENOMEM);
 610                 }
 611         }
 612
 613         if (t->fs.newsmac) {
 614                 /* XXX: alloc SMT */
 615                 return (ENOTSUP);
 616         }
 617
 618         if (t->fs.hash)
 619                 return (set_hashfilter(sc, t, l2te));
 620         else
 621                 return (set_tcamfilter(sc, t, l2te));
 622
 623 }
 624
 625 static int
 626 del_tcamfilter(struct adapter *sc, struct t4_filter *t)
 627 {
 628         struct filter_entry *f;
 629         struct fw_filter_wr *fwr;
 630         struct wrq_cookie cookie;
 631         int rc;
 632
 633         MPASS(sc->tids.ftid_tab != NULL);
 634         MPASS(sc->tids.nftids > 0);
 635
 636         if (t->idx >= sc->tids.nftids)
 637                 return (EINVAL);
 638
 639         mtx_lock(&sc->tids.ftid_lock);
 640         f = &sc->tids.ftid_tab[t->idx];
 641         if (f->locked) {
 642                 rc = EPERM;
 643                 goto done;
 644         }
 645         if (f->pending) {
 646                 rc = EBUSY;
 647                 goto done;
 648         }
 649         if (f->valid == 0) {
 650                 rc = EINVAL;
 651                 goto done;
 652         }
 653         MPASS(f->tid == sc->tids.ftid_base + t->idx);
 654         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
 655         if (fwr == NULL) {
 656                 rc = ENOMEM;
 657                 goto done;
 658         }
 659
 660         bzero(fwr, sizeof (*fwr));
 661         t4_mk_filtdelwr(f->tid, fwr, sc->sge.fwq.abs_id);
 662         f->pending = 1;
 663         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
 664         t->fs = f->fs;  /* extra info for the caller */
 665
 666         for (;;) {
 667                 if (f->pending == 0) {
 668                         rc = f->valid ? EIO : 0;
 669                         break;
 670                 }
 671                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
 672                         rc = EINPROGRESS;
 673                         break;
 674                 }
 675         }
 676 done:
 677         mtx_unlock(&sc->tids.ftid_lock);
 678         return (rc);
 679 }
 680
 681 int
 682 del_filter(struct adapter *sc, struct t4_filter *t)
 683 {
 684
 685         /* No filters possible if not initialized yet. */
 686         if (!(sc->flags & FULL_INIT_DONE))
 687                 return (EINVAL);
 688
 689         /*
 690          * The checks for tid tables ensure that the locks that del_* will reach
 691          * for are initialized.
 692          */
 693         if (t->fs.hash) {
 694                 if (sc->tids.hftid_tab != NULL)
 695                         return (del_hashfilter(sc, t));
 696         } else {
 697                 if (sc->tids.ftid_tab != NULL)
 698                         return (del_tcamfilter(sc, t));
 699         }
 700
 701         return (EINVAL);
 702 }
 703
 704 /*
 705  * Release secondary resources associated with the filter.
 706  */
 707 static void
 708 free_filter_resources(struct filter_entry *f)
 709 {
 710
 711         if (f->l2te) {
 712                 t4_l2t_release(f->l2te);
 713                 f->l2te = NULL;
 714         }
 715 }
 716
 717 int
 718 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 719 {
 720         struct adapter *sc = iq->adapter;
 721         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
 722         u_int tid = GET_TID(rpl);
 723         u_int rc, cleanup, idx;
 724         struct filter_entry *f;
 725
 726         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
 727             rss->opcode));
 728         MPASS(is_ftid(sc, tid));
 729
 730         cleanup = 0;
 731         idx = tid - sc->tids.ftid_base;
 732         f = &sc->tids.ftid_tab[idx];
 733         rc = G_COOKIE(rpl->cookie);
 734
 735         mtx_lock(&sc->tids.ftid_lock);
 736         KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.",
 737             __func__, rc, idx));
 738         switch(rc) {
 739         case FW_FILTER_WR_FLT_ADDED:
 740                 /* set-filter succeeded */
 741                 f->valid = 1;
 742                 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
 743                 break;
 744         case FW_FILTER_WR_FLT_DELETED:
 745                 /* del-filter succeeded */
 746                 MPASS(f->valid == 1);
 747                 f->valid = 0;
 748                 /* Fall through */
 749         case FW_FILTER_WR_SMT_TBL_FULL:
 750                 /* set-filter failed due to lack of SMT space. */
 751                 MPASS(f->valid == 0);
 752                 free_filter_resources(f);
 753                 sc->tids.ftids_in_use--;
 754                 break;
 755         case FW_FILTER_WR_SUCCESS:
 756         case FW_FILTER_WR_EINVAL:
 757         default:
 758                 panic("%s: unexpected reply %d for filter[%d].", __func__, rc,
 759                     idx);
 760         }
 761         f->pending = 0;
 762         cv_broadcast(&sc->tids.ftid_cv);
 763         mtx_unlock(&sc->tids.ftid_lock);
 764
 765         return (0);
 766 }
 767
 768 /*
 769  * This is the reply to the Active Open that created the filter.  Additional TCB
 770  * updates may be required to complete the filter configuration.
 771  */
 772 int
 773 t4_hashfilter_ao_rpl(struct sge_iq *iq, const struct rss_header *rss,
 774     struct mbuf *m)
 775 {
 776         struct adapter *sc = iq->adapter;
 777         const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
 778         u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
 779         u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
 780         struct filter_entry *f = lookup_atid(sc, atid);
 781
 782         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 783
 784         mtx_lock(&sc->tids.hftid_lock);
 785         KASSERT(f->pending, ("%s: hashfilter[%p] isn't pending.", __func__, f));
 786         KASSERT(f->tid == -1, ("%s: hashfilter[%p] has tid %d already.",
 787             __func__, f, f->tid));
 788         if (status == CPL_ERR_NONE) {
 789                 struct filter_entry *f2;
 790
 791                 f->tid = GET_TID(cpl);
 792                 MPASS(f->tid < sc->tids.ntids);
 793                 if (__predict_false((f2 = lookup_hftid(sc, f->tid)) != NULL)) {
 794                         /* XXX: avoid hash collisions in the first place. */
 795                         MPASS(f2->tid == f->tid);
 796                         remove_hftid(sc, f2->tid, f2->fs.type ? 2 : 1);
 797                         free_filter_resources(f2);
 798                         free(f2, M_CXGBE);
 799                 }
 800                 insert_hftid(sc, f->tid, f, f->fs.type ? 2 : 1);
 801                 /*
 802                  * Leave the filter pending until it is fully set up, which will
 803                  * be indicated by the reply to the last TCB update.  No need to
 804                  * unblock the ioctl thread either.
 805                  */
 806                 if (configure_hashfilter_tcb(sc, f) == EINPROGRESS)
 807                         goto done;
 808                 f->valid = 1;
 809                 f->pending = 0;
 810         } else {
 811                 /* provide errno instead of tid to ioctl */
 812                 f->tid = act_open_rpl_status_to_errno(status);
 813                 f->valid = 0;
 814                 if (act_open_has_tid(status))
 815                         release_tid(sc, GET_TID(cpl), &sc->sge.mgmtq);
 816                 free_filter_resources(f);
 817                 if (f->locked == 0)
 818                         free(f, M_CXGBE);
 819         }
 820         cv_broadcast(&sc->tids.hftid_cv);
 821 done:
 822         mtx_unlock(&sc->tids.hftid_lock);
 823
 824         free_atid(sc, atid);
 825         return (0);
 826 }
 827
 828 int
 829 t4_hashfilter_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss,
 830     struct mbuf *m)
 831 {
 832         struct adapter *sc = iq->adapter;
 833         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
 834         u_int tid = GET_TID(rpl);
 835         struct filter_entry *f;
 836
 837         mtx_lock(&sc->tids.hftid_lock);
 838         f = lookup_hftid(sc, tid);
 839         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
 840         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
 841             f, tid));
 842         KASSERT(f->valid == 0, ("%s: hashfilter %p [%u] is valid already.",
 843             __func__, f, tid));
 844         f->pending = 0;
 845         if (rpl->status == 0) {
 846                 f->valid = 1;
 847         } else {
 848                 f->tid = EIO;
 849                 f->valid = 0;
 850                 free_filter_resources(f);
 851                 remove_hftid(sc, tid, f->fs.type ? 2 : 1);
 852                 release_tid(sc, tid, &sc->sge.mgmtq);
 853                 if (f->locked == 0)
 854                         free(f, M_CXGBE);
 855         }
 856         cv_broadcast(&sc->tids.hftid_cv);
 857         mtx_unlock(&sc->tids.hftid_lock);
 858
 859         return (0);
 860 }
 861
 862 int
 863 t4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss,
 864     struct mbuf *m)
 865 {
 866         struct adapter *sc = iq->adapter;
 867         const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
 868         unsigned int tid = GET_TID(cpl);
 869         struct filter_entry *f;
 870
 871         mtx_lock(&sc->tids.hftid_lock);
 872         f = lookup_hftid(sc, tid);
 873         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
 874         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
 875             f, tid));
 876         KASSERT(f->valid, ("%s: hashfilter %p [%u] isn't valid.", __func__, f,
 877             tid));
 878         f->pending = 0;
 879         if (cpl->status == 0) {
 880                 f->valid = 0;
 881                 free_filter_resources(f);
 882                 remove_hftid(sc, tid, f->fs.type ? 2 : 1);
 883                 release_tid(sc, tid, &sc->sge.mgmtq);
 884                 if (f->locked == 0)
 885                         free(f, M_CXGBE);
 886         }
 887         cv_broadcast(&sc->tids.hftid_cv);
 888         mtx_unlock(&sc->tids.hftid_lock);
 889
 890         return (0);
 891 }
 892
 893 static int
 894 get_hashfilter(struct adapter *sc, struct t4_filter *t)
 895 {
 896         int i, nfilters = sc->tids.ntids;
 897         struct filter_entry *f;
 898
 899         if (sc->tids.tids_in_use == 0 || sc->tids.hftid_tab == NULL ||
 900             t->idx >= nfilters) {
 901                 t->idx = 0xffffffff;
 902                 return (0);
 903         }
 904
 905         mtx_lock(&sc->tids.hftid_lock);
 906         for (i = t->idx; i < nfilters; i++) {
 907                 f = lookup_hftid(sc, i);
 908                 if (f != NULL && f->valid) {
 909                         t->idx = i;
 910                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
 911                         t->smtidx = f->smtidx;
 912                         if (f->fs.hitcnts)
 913                                 t->hits = get_filter_hits(sc, t->idx);
 914                         else
 915                                 t->hits = UINT64_MAX;
 916                         t->fs = f->fs;
 917
 918                         goto done;
 919                 }
 920         }
 921         t->idx = 0xffffffff;
 922 done:
 923         mtx_unlock(&sc->tids.hftid_lock);
 924         return (0);
 925 }
 926
 927 static uint64_t
 928 hashfilter_ntuple(struct adapter *sc, const struct t4_filter_specification *fs)
 929 {
 930         struct tp_params *tp = &sc->params.tp;
 931         uint64_t ntuple = 0;
 932
 933         /*
 934          * Initialize each of the fields which we care about which are present
 935          * in the Compressed Filter Tuple.
 936          */
 937         if (tp->vlan_shift >= 0 && fs->mask.vlan)
 938                 ntuple |= (F_FT_VLAN_VLD | fs->val.vlan) << tp->vlan_shift;
 939
 940         if (tp->port_shift >= 0 && fs->mask.iport)
 941                 ntuple |= (uint64_t)fs->val.iport << tp->port_shift;
 942
 943         if (tp->protocol_shift >= 0) {
 944                 if (!fs->val.proto)
 945                         ntuple |= (uint64_t)IPPROTO_TCP << tp->protocol_shift;
 946                 else
 947                         ntuple |= (uint64_t)fs->val.proto << tp->protocol_shift;
 948         }
 949
 950         if (tp->tos_shift >= 0 && fs->mask.tos)
 951                 ntuple |= (uint64_t)(fs->val.tos) << tp->tos_shift;
 952
 953         if (tp->vnic_shift >= 0) {
 954 #ifdef notyet
 955                 if (tp->ingress_config & F_VNIC && fs->mask.pfvf_vld)
 956                         ntuple |= (uint64_t)((fs->val.pfvf_vld << 16) |
 957                                         (fs->val.pf << 13) |
 958                                         (fs->val.vf)) << tp->vnic_shift;
 959                 else
 960 #endif
 961                         ntuple |= (uint64_t)((fs->val.ovlan_vld << 16) |
 962                                         (fs->val.vnic)) << tp->vnic_shift;
 963         }
 964
 965         if (tp->macmatch_shift >= 0 && fs->mask.macidx)
 966                 ntuple |= (uint64_t)(fs->val.macidx) << tp->macmatch_shift;
 967
 968         if (tp->ethertype_shift >= 0 && fs->mask.ethtype)
 969                 ntuple |= (uint64_t)(fs->val.ethtype) << tp->ethertype_shift;
 970
 971         if (tp->matchtype_shift >= 0 && fs->mask.matchtype)
 972                 ntuple |= (uint64_t)(fs->val.matchtype) << tp->matchtype_shift;
 973
 974         if (tp->frag_shift >= 0 && fs->mask.frag)
 975                 ntuple |= (uint64_t)(fs->val.frag) << tp->frag_shift;
 976
 977         if (tp->fcoe_shift >= 0 && fs->mask.fcoe)
 978                 ntuple |= (uint64_t)(fs->val.fcoe) << tp->fcoe_shift;
 979
 980         return (ntuple);
 981 }
 982
 983 static void
 984 mk_act_open_req6(struct adapter *sc, struct filter_entry *f, int atid,
 985     struct cpl_act_open_req6 *cpl)
 986 {
 987         struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
 988         struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
 989
 990         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
 991         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
 992         MPASS(atid >= 0);
 993
 994         if (chip_id(sc) == CHELSIO_T5) {
 995                 INIT_TP_WR(cpl5, 0);
 996         } else {
 997                 INIT_TP_WR(cpl6, 0);
 998                 cpl6->rsvd2 = 0;
 999                 cpl6->opt3 = 0;
1000         }
1001
1002         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
1003             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1004             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1005         cpl->local_port = htobe16(f->fs.val.dport);
1006         cpl->peer_port = htobe16(f->fs.val.sport);
1007         cpl->local_ip_hi = *(uint64_t *)(&f->fs.val.dip);
1008         cpl->local_ip_lo = *(((uint64_t *)&f->fs.val.dip) + 1);
1009         cpl->peer_ip_hi = *(uint64_t *)(&f->fs.val.sip);
1010         cpl->peer_ip_lo = *(((uint64_t *)&f->fs.val.sip) + 1);
1011         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1012             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1013             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1014             V_NO_CONG(f->fs.rpttid) | F_TCAM_BYPASS | F_NON_OFFLOAD);
1015
1016         cpl6->params = htobe64(V_FILTER_TUPLE(hashfilter_ntuple(sc, &f->fs)));
1017         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1018             F_T5_OPT_2_VALID | F_RX_CHANNEL |
1019             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1020             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1021 }
1022
1023 static void
1024 mk_act_open_req(struct adapter *sc, struct filter_entry *f, int atid,
1025     struct cpl_act_open_req *cpl)
1026 {
1027         struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
1028         struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
1029
1030         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1031         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1032         MPASS(atid >= 0);
1033
1034         if (chip_id(sc) == CHELSIO_T5) {
1035                 INIT_TP_WR(cpl5, 0);
1036         } else {
1037                 INIT_TP_WR(cpl6, 0);
1038                 cpl6->rsvd2 = 0;
1039                 cpl6->opt3 = 0;
1040         }
1041
1042         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
1043             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1044             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1045         cpl->local_port = htobe16(f->fs.val.dport);
1046         cpl->peer_port = htobe16(f->fs.val.sport);
1047         cpl->local_ip = f->fs.val.dip[0] | f->fs.val.dip[1] << 8 |
1048             f->fs.val.dip[2] << 16 | f->fs.val.dip[3] << 24;
1049         cpl->peer_ip = f->fs.val.sip[0] | f->fs.val.sip[1] << 8 |
1050                 f->fs.val.sip[2] << 16 | f->fs.val.sip[3] << 24;
1051         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1052             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1053             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1054             V_NO_CONG(f->fs.rpttid) | F_TCAM_BYPASS | F_NON_OFFLOAD);
1055
1056         cpl6->params = htobe64(V_FILTER_TUPLE(hashfilter_ntuple(sc, &f->fs)));
1057         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1058             F_T5_OPT_2_VALID | F_RX_CHANNEL |
1059             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1060             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1061 }
1062
1063 static int
1064 act_open_cpl_len16(struct adapter *sc, int isipv6)
1065 {
1066         int idx;
1067         static const int sz_table[3][2] = {
1068                 {
1069                         howmany(sizeof (struct cpl_act_open_req), 16),
1070                         howmany(sizeof (struct cpl_act_open_req6), 16)
1071                 },
1072                 {
1073                         howmany(sizeof (struct cpl_t5_act_open_req), 16),
1074                         howmany(sizeof (struct cpl_t5_act_open_req6), 16)
1075                 },
1076                 {
1077                         howmany(sizeof (struct cpl_t6_act_open_req), 16),
1078                         howmany(sizeof (struct cpl_t6_act_open_req6), 16)
1079                 },
1080         };
1081
1082         MPASS(chip_id(sc) >= CHELSIO_T4);
1083         idx = min(chip_id(sc) - CHELSIO_T4, 2);
1084
1085         return (sz_table[idx][!!isipv6]);
1086 }
1087
1088 static int
1089 set_hashfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te)
1090 {
1091         void *wr;
1092         struct wrq_cookie cookie;
1093         struct filter_entry *f;
1094         int rc, atid = -1;
1095
1096         MPASS(t->fs.hash);
1097         /* Already validated against fconf, iconf */
1098         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
1099         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
1100
1101         mtx_lock(&sc->tids.hftid_lock);
1102
1103         /*
1104          * XXX: Check for hash collisions and insert in the hash based lookup
1105          * table so that in-flight hashfilters are also considered when checking
1106          * for collisions.
1107          */
1108
1109         f = malloc(sizeof(*f), M_CXGBE, M_ZERO | M_NOWAIT);
1110         if (__predict_false(f == NULL)) {
1111                 if (l2te)
1112                         t4_l2t_release(l2te);
1113                 rc = ENOMEM;
1114                 goto done;
1115         }
1116         f->fs = t->fs;
1117         f->l2te = l2te;
1118
1119         atid = alloc_atid(sc, f);
1120         if (__predict_false(atid) == -1) {
1121                 if (l2te)
1122                         t4_l2t_release(l2te);
1123                 free(f, M_CXGBE);
1124                 rc = EAGAIN;
1125                 goto done;
1126         }
1127         MPASS(atid >= 0);
1128
1129         wr = start_wrq_wr(&sc->sge.mgmtq, act_open_cpl_len16(sc, f->fs.type),
1130             &cookie);
1131         if (wr == NULL) {
1132                 free_atid(sc, atid);
1133                 if (l2te)
1134                         t4_l2t_release(l2te);
1135                 free(f, M_CXGBE);
1136                 rc = ENOMEM;
1137                 goto done;
1138         }
1139         if (f->fs.type)
1140                 mk_act_open_req6(sc, f, atid, wr);
1141         else
1142                 mk_act_open_req(sc, f, atid, wr);
1143
1144         f->locked = 1; /* ithread mustn't free f if ioctl is still around. */
1145         f->pending = 1;
1146         f->tid = -1;
1147         commit_wrq_wr(&sc->sge.mgmtq, wr, &cookie);
1148
1149         for (;;) {
1150                 MPASS(f->locked);
1151                 if (f->pending == 0) {
1152                         if (f->valid) {
1153                                 rc = 0;
1154                                 f->locked = 0;
1155                                 t->idx = f->tid;
1156                         } else {
1157                                 rc = f->tid;
1158                                 free(f, M_CXGBE);
1159                         }
1160                         break;
1161                 }
1162                 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1163                         f->locked = 0;
1164                         rc = EINPROGRESS;
1165                         break;
1166                 }
1167         }
1168 done:
1169         mtx_unlock(&sc->tids.hftid_lock);
1170         return (rc);
1171 }
1172
1173 /* SET_TCB_FIELD sent as a ULP command looks like this */
1174 #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \
1175     sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core))
1176
1177 static void *
1178 mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, uint64_t word, uint64_t mask,
1179                 uint64_t val, uint32_t tid, uint32_t qid)
1180 {
1181         struct ulptx_idata *ulpsc;
1182         struct cpl_set_tcb_field_core *req;
1183
1184         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1185         ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
1186
1187         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1188         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1189         ulpsc->len = htobe32(sizeof(*req));
1190
1191         req = (struct cpl_set_tcb_field_core *)(ulpsc + 1);
1192         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1193         req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(qid));
1194         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
1195         req->mask = htobe64(mask);
1196         req->val = htobe64(val);
1197
1198         ulpsc = (struct ulptx_idata *)(req + 1);
1199         if (LEN__SET_TCB_FIELD_ULP % 16) {
1200                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1201                 ulpsc->len = htobe32(0);
1202                 return (ulpsc + 1);
1203         }
1204         return (ulpsc);
1205 }
1206
1207 /* ABORT_REQ sent as a ULP command looks like this */
1208 #define LEN__ABORT_REQ_ULP (sizeof(struct ulp_txpkt) + \
1209         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_req_core))
1210
1211 static void *
1212 mk_abort_req_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1213 {
1214         struct ulptx_idata *ulpsc;
1215         struct cpl_abort_req_core *req;
1216
1217         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1218         ulpmc->len = htobe32(howmany(LEN__ABORT_REQ_ULP, 16));
1219
1220         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1221         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1222         ulpsc->len = htobe32(sizeof(*req));
1223
1224         req = (struct cpl_abort_req_core *)(ulpsc + 1);
1225         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
1226         req->rsvd0 = htonl(0);
1227         req->rsvd1 = 0;
1228         req->cmd = CPL_ABORT_NO_RST;
1229
1230         ulpsc = (struct ulptx_idata *)(req + 1);
1231         if (LEN__ABORT_REQ_ULP % 16) {
1232                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1233                 ulpsc->len = htobe32(0);
1234                 return (ulpsc + 1);
1235         }
1236         return (ulpsc);
1237 }
1238
1239 /* ABORT_RPL sent as a ULP command looks like this */
1240 #define LEN__ABORT_RPL_ULP (sizeof(struct ulp_txpkt) + \
1241         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_rpl_core))
1242
1243 static void *
1244 mk_abort_rpl_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1245 {
1246         struct ulptx_idata *ulpsc;
1247         struct cpl_abort_rpl_core *rpl;
1248
1249         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1250         ulpmc->len = htobe32(howmany(LEN__ABORT_RPL_ULP, 16));
1251
1252         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1253         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1254         ulpsc->len = htobe32(sizeof(*rpl));
1255
1256         rpl = (struct cpl_abort_rpl_core *)(ulpsc + 1);
1257         OPCODE_TID(rpl) = htobe32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
1258         rpl->rsvd0 = htonl(0);
1259         rpl->rsvd1 = 0;
1260         rpl->cmd = CPL_ABORT_NO_RST;
1261
1262         ulpsc = (struct ulptx_idata *)(rpl + 1);
1263         if (LEN__ABORT_RPL_ULP % 16) {
1264                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1265                 ulpsc->len = htobe32(0);
1266                 return (ulpsc + 1);
1267         }
1268         return (ulpsc);
1269 }
1270
1271 static inline int
1272 del_hashfilter_wrlen(void)
1273 {
1274
1275         return (sizeof(struct work_request_hdr) +
1276             roundup2(LEN__SET_TCB_FIELD_ULP, 16) +
1277             roundup2(LEN__ABORT_REQ_ULP, 16) +
1278             roundup2(LEN__ABORT_RPL_ULP, 16));
1279 }
1280
1281 static void
1282 mk_del_hashfilter_wr(int tid, struct work_request_hdr *wrh, int wrlen, int qid)
1283 {
1284         struct ulp_txpkt *ulpmc;
1285
1286         INIT_ULPTX_WRH(wrh, wrlen, 0, 0);
1287         ulpmc = (struct ulp_txpkt *)(wrh + 1);
1288         ulpmc = mk_set_tcb_field_ulp(ulpmc, W_TCB_RSS_INFO,
1289             V_TCB_RSS_INFO(M_TCB_RSS_INFO), V_TCB_RSS_INFO(qid), tid, 0);
1290         ulpmc = mk_abort_req_ulp(ulpmc, tid);
1291         ulpmc = mk_abort_rpl_ulp(ulpmc, tid);
1292 }
1293
1294 static int
1295 del_hashfilter(struct adapter *sc, struct t4_filter *t)
1296 {
1297         void *wr;
1298         struct filter_entry *f;
1299         struct wrq_cookie cookie;
1300         int rc;
1301         const int wrlen = del_hashfilter_wrlen();
1302
1303         MPASS(sc->tids.hftid_tab != NULL);
1304         MPASS(sc->tids.ntids > 0);
1305
1306         if (t->idx >= sc->tids.ntids)
1307                 return (EINVAL);
1308
1309         mtx_lock(&sc->tids.hftid_lock);
1310         f = lookup_hftid(sc, t->idx);
1311         if (f == NULL || f->valid == 0) {
1312                 rc = EINVAL;
1313                 goto done;
1314         }
1315         MPASS(f->tid == t->idx);
1316         if (f->locked) {
1317                 rc = EPERM;
1318                 goto done;
1319         }
1320         if (f->pending) {
1321                 rc = EBUSY;
1322                 goto done;
1323         }
1324         wr = start_wrq_wr(&sc->sge.mgmtq, howmany(wrlen, 16), &cookie);
1325         if (wr == NULL) {
1326                 rc = ENOMEM;
1327                 goto done;
1328         }
1329
1330         mk_del_hashfilter_wr(t->idx, wr, wrlen, sc->sge.fwq.abs_id);
1331         f->locked = 1;
1332         f->pending = 1;
1333         commit_wrq_wr(&sc->sge.mgmtq, wr, &cookie);
1334         t->fs = f->fs;  /* extra info for the caller */
1335
1336         for (;;) {
1337                 MPASS(f->locked);
1338                 if (f->pending == 0) {
1339                         if (f->valid) {
1340                                 f->locked = 0;
1341                                 rc = EIO;
1342                         } else {
1343                                 rc = 0;
1344                                 free(f, M_CXGBE);
1345                         }
1346                         break;
1347                 }
1348                 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1349                         f->locked = 0;
1350                         rc = EINPROGRESS;
1351                         break;
1352                 }
1353         }
1354 done:
1355         mtx_unlock(&sc->tids.hftid_lock);
1356         return (rc);
1357 }
1358
1359 static int
1360 set_tcb_field(struct adapter *sc, u_int tid, uint16_t word, uint64_t mask,
1361     uint64_t val, int no_reply)
1362 {
1363         struct wrq_cookie cookie;
1364         struct cpl_set_tcb_field *req;
1365
1366         req = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*req), 16), &cookie);
1367         if (req == NULL)
1368                 return (ENOMEM);
1369         bzero(req, sizeof(*req));
1370         INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid);
1371         if (no_reply == 0) {
1372                 req->reply_ctrl = htobe16(V_QUEUENO(sc->sge.fwq.abs_id) |
1373                     V_NO_REPLY(0));
1374         } else
1375                 req->reply_ctrl = htobe16(V_NO_REPLY(1));
1376         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(CPL_COOKIE_HASHFILTER));
1377         req->mask = htobe64(mask);
1378         req->val = htobe64(val);
1379         commit_wrq_wr(&sc->sge.mgmtq, req, &cookie);
1380
1381         return (0);
1382 }
1383
1384 /* Set one of the t_flags bits in the TCB. */
1385 static inline int
1386 set_tcb_tflag(struct adapter *sc, int tid, u_int bit_pos, u_int val)
1387 {
1388
1389         return (set_tcb_field(sc, tid,  W_TCB_T_FLAGS, 1ULL << bit_pos,
1390             (uint64_t)val << bit_pos, 1));
1391 }
1392
1393 /*
1394  * Returns EINPROGRESS to indicate that at least one TCB update was sent and the
1395  * last of the series of updates requested a reply.  The reply informs the
1396  * driver that the filter is fully setup.
1397  */
1398 static int
1399 configure_hashfilter_tcb(struct adapter *sc, struct filter_entry *f)
1400 {
1401         int updated = 0;
1402
1403         MPASS(f->tid < sc->tids.ntids);
1404         MPASS(f->fs.hash);
1405         MPASS(f->pending);
1406         MPASS(f->valid == 0);
1407
1408         if (f->fs.newdmac) {
1409                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECE, 1);
1410                 updated++;
1411         }
1412
1413         if (f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) {
1414                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_RFR, 1);
1415                 updated++;
1416         }
1417
1418         if (f->fs.hitcnts || updated > 0) {
1419                 set_tcb_field(sc, f->tid, W_TCB_TIMESTAMP,
1420                     V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) |
1421                     V_TCB_T_RTT_TS_RECENT_AGE(M_TCB_T_RTT_TS_RECENT_AGE),
1422                     V_TCB_TIMESTAMP(0ULL) | V_TCB_T_RTT_TS_RECENT_AGE(0ULL), 0);
1423                 return (EINPROGRESS);
1424         }
1425
1426         return (0);
1427 }