sys/dev/cxgbe/t4_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2018 Chelsio Communications, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/fnv_hash.h>
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/module.h>
  40 #include <sys/bus.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/rwlock.h>
  44 #include <sys/socket.h>
  45 #include <sys/sbuf.h>
  46 #include <netinet/in.h>
  47
  48 #include "common/common.h"
  49 #include "common/t4_msg.h"
  50 #include "common/t4_regs.h"
  51 #include "common/t4_regs_values.h"
  52 #include "common/t4_tcb.h"
  53 #include "t4_l2t.h"
  54 #include "t4_smt.h"
  55
  56 struct filter_entry {
  57         LIST_ENTRY(filter_entry) link_4t;
  58         LIST_ENTRY(filter_entry) link_tid;
  59
  60         uint32_t valid:1;       /* filter allocated and valid */
  61         uint32_t locked:1;      /* filter is administratively locked or busy */
  62         uint32_t pending:1;     /* filter action is pending firmware reply */
  63         int tid;                /* tid of the filter TCB */
  64         struct l2t_entry *l2te; /* L2 table entry for DMAC rewrite */
  65         struct smt_entry *smt;  /* SMT entry for SMAC rewrite */
  66
  67         struct t4_filter_specification fs;
  68 };
  69
  70 static void free_filter_resources(struct filter_entry *);
  71 static int get_tcamfilter(struct adapter *, struct t4_filter *);
  72 static int get_hashfilter(struct adapter *, struct t4_filter *);
  73 static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t,
  74     struct l2t_entry *, struct smt_entry *);
  75 static int del_hashfilter(struct adapter *, struct t4_filter *);
  76 static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *);
  77
  78 static inline bool
  79 separate_hpfilter_region(struct adapter *sc)
  80 {
  81
  82         return (chip_id(sc) >= CHELSIO_T6);
  83 }
  84
  85 static inline uint32_t
  86 hf_hashfn_4t(struct t4_filter_specification *fs)
  87 {
  88         struct t4_filter_tuple *ft = &fs->val;
  89         uint32_t hash;
  90
  91         if (fs->type) {
  92                 /* IPv6 */
  93                 hash = fnv_32_buf(&ft->sip[0], 16, FNV1_32_INIT);
  94                 hash = fnv_32_buf(&ft->dip[0], 16, hash);
  95         } else {
  96                 hash = fnv_32_buf(&ft->sip[0], 4, FNV1_32_INIT);
  97                 hash = fnv_32_buf(&ft->dip[0], 4, hash);
  98         }
  99         hash = fnv_32_buf(&ft->sport, sizeof(ft->sport), hash);
 100         hash = fnv_32_buf(&ft->dport, sizeof(ft->dport), hash);
 101
 102         return (hash);
 103 }
 104
 105 static inline uint32_t
 106 hf_hashfn_tid(int tid)
 107 {
 108
 109         return (fnv_32_buf(&tid, sizeof(tid), FNV1_32_INIT));
 110 }
 111
 112 static int
 113 alloc_hftid_hash(struct tid_info *t, int flags)
 114 {
 115         int n;
 116
 117         MPASS(t->ntids > 0);
 118         MPASS(t->hftid_hash_4t == NULL);
 119         MPASS(t->hftid_hash_tid == NULL);
 120
 121         n = max(t->ntids / 1024, 16);
 122         t->hftid_hash_4t = hashinit_flags(n, M_CXGBE, &t->hftid_4t_mask, flags);
 123         if (t->hftid_hash_4t == NULL)
 124                 return (ENOMEM);
 125         t->hftid_hash_tid = hashinit_flags(n, M_CXGBE, &t->hftid_tid_mask,
 126             flags);
 127         if (t->hftid_hash_tid == NULL) {
 128                 hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask);
 129                 t->hftid_hash_4t = NULL;
 130                 return (ENOMEM);
 131         }
 132
 133         mtx_init(&t->hftid_lock, "T4 hashfilters", 0, MTX_DEF);
 134         cv_init(&t->hftid_cv, "t4hfcv");
 135
 136         return (0);
 137 }
 138
 139 void
 140 free_hftid_hash(struct tid_info *t)
 141 {
 142         struct filter_entry *f, *ftmp;
 143         LIST_HEAD(, filter_entry) *head;
 144         int i;
 145 #ifdef INVARIANTS
 146         int n = 0;
 147 #endif
 148
 149         if (t->tids_in_use > 0) {
 150                 /* Remove everything from the tid hash. */
 151                 head = t->hftid_hash_tid;
 152                 for (i = 0; i <= t->hftid_tid_mask; i++) {
 153                         LIST_FOREACH_SAFE(f, &head[i], link_tid, ftmp) {
 154                                 LIST_REMOVE(f, link_tid);
 155                         }
 156                 }
 157
 158                 /* Remove and then free each filter in the 4t hash. */
 159                 head = t->hftid_hash_4t;
 160                 for (i = 0; i <= t->hftid_4t_mask; i++) {
 161                         LIST_FOREACH_SAFE(f, &head[i], link_4t, ftmp) {
 162 #ifdef INVARIANTS
 163                                 n += f->fs.type ? 2 : 1;
 164 #endif
 165                                 LIST_REMOVE(f, link_4t);
 166                                 free(f, M_CXGBE);
 167                         }
 168                 }
 169                 MPASS(t->tids_in_use == n);
 170                 t->tids_in_use = 0;
 171         }
 172
 173         if (t->hftid_hash_4t) {
 174                 hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask);
 175                 t->hftid_hash_4t = NULL;
 176         }
 177         if (t->hftid_hash_tid) {
 178                 hashdestroy(t->hftid_hash_tid, M_CXGBE, t->hftid_tid_mask);
 179                 t->hftid_hash_tid = NULL;
 180         }
 181         if (mtx_initialized(&t->hftid_lock)) {
 182                 mtx_destroy(&t->hftid_lock);
 183                 cv_destroy(&t->hftid_cv);
 184         }
 185 }
 186
 187 static void
 188 insert_hf(struct adapter *sc, struct filter_entry *f, uint32_t hash)
 189 {
 190         struct tid_info *t = &sc->tids;
 191         LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t;
 192
 193         MPASS(head != NULL);
 194         if (hash == 0)
 195                 hash = hf_hashfn_4t(&f->fs);
 196         LIST_INSERT_HEAD(&head[hash & t->hftid_4t_mask], f, link_4t);
 197         atomic_add_int(&t->tids_in_use, f->fs.type ? 2 : 1);
 198 }
 199
 200 static void
 201 insert_hftid(struct adapter *sc, struct filter_entry *f)
 202 {
 203         struct tid_info *t = &sc->tids;
 204         LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid;
 205         uint32_t hash;
 206
 207         MPASS(f->tid >= t->tid_base);
 208         MPASS(f->tid - t->tid_base < t->ntids);
 209         mtx_assert(&t->hftid_lock, MA_OWNED);
 210
 211         hash = hf_hashfn_tid(f->tid);
 212         LIST_INSERT_HEAD(&head[hash & t->hftid_tid_mask], f, link_tid);
 213 }
 214
 215 static bool
 216 filter_eq(struct t4_filter_specification *fs1,
 217     struct t4_filter_specification *fs2)
 218 {
 219         int n;
 220
 221         MPASS(fs1->hash && fs2->hash);
 222
 223         if (fs1->type != fs2->type)
 224                 return (false);
 225
 226         n = fs1->type ? 16 : 4;
 227         if (bcmp(&fs1->val.sip[0], &fs2->val.sip[0], n) ||
 228             bcmp(&fs1->val.dip[0], &fs2->val.dip[0], n) ||
 229             fs1->val.sport != fs2->val.sport ||
 230             fs1->val.dport != fs2->val.dport)
 231                 return (false);
 232
 233         /*
 234          * We know the masks are the same because all hashfilters conform to the
 235          * global tp->filter_mask and the driver has verified that already.
 236          */
 237
 238         if ((fs1->mask.pfvf_vld || fs1->mask.ovlan_vld) &&
 239             fs1->val.vnic != fs2->val.vnic)
 240                 return (false);
 241         if (fs1->mask.vlan_vld && fs1->val.vlan != fs2->val.vlan)
 242                 return (false);
 243         if (fs1->mask.macidx && fs1->val.macidx != fs2->val.macidx)
 244                 return (false);
 245         if (fs1->mask.frag && fs1->val.frag != fs2->val.frag)
 246                 return (false);
 247         if (fs1->mask.matchtype && fs1->val.matchtype != fs2->val.matchtype)
 248                 return (false);
 249         if (fs1->mask.iport && fs1->val.iport != fs2->val.iport)
 250                 return (false);
 251         if (fs1->mask.fcoe && fs1->val.fcoe != fs2->val.fcoe)
 252                 return (false);
 253         if (fs1->mask.proto && fs1->val.proto != fs2->val.proto)
 254                 return (false);
 255         if (fs1->mask.tos && fs1->val.tos != fs2->val.tos)
 256                 return (false);
 257         if (fs1->mask.ethtype && fs1->val.ethtype != fs2->val.ethtype)
 258                 return (false);
 259
 260         return (true);
 261 }
 262
 263 static struct filter_entry *
 264 lookup_hf(struct adapter *sc, struct t4_filter_specification *fs, uint32_t hash)
 265 {
 266         struct tid_info *t = &sc->tids;
 267         LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t;
 268         struct filter_entry *f;
 269
 270         mtx_assert(&t->hftid_lock, MA_OWNED);
 271         MPASS(head != NULL);
 272
 273         if (hash == 0)
 274                 hash = hf_hashfn_4t(fs);
 275
 276         LIST_FOREACH(f, &head[hash & t->hftid_4t_mask], link_4t) {
 277                 if (filter_eq(&f->fs, fs))
 278                         return (f);
 279         }
 280
 281         return (NULL);
 282 }
 283
 284 static struct filter_entry *
 285 lookup_hftid(struct adapter *sc, int tid)
 286 {
 287         struct tid_info *t = &sc->tids;
 288         LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid;
 289         struct filter_entry *f;
 290         uint32_t hash;
 291
 292         mtx_assert(&t->hftid_lock, MA_OWNED);
 293         MPASS(head != NULL);
 294
 295         hash = hf_hashfn_tid(tid);
 296         LIST_FOREACH(f, &head[hash & t->hftid_tid_mask], link_tid) {
 297                 if (f->tid == tid)
 298                         return (f);
 299         }
 300
 301         return (NULL);
 302 }
 303
 304 static void
 305 remove_hf(struct adapter *sc, struct filter_entry *f)
 306 {
 307         struct tid_info *t = &sc->tids;
 308
 309         mtx_assert(&t->hftid_lock, MA_OWNED);
 310
 311         LIST_REMOVE(f, link_4t);
 312         atomic_subtract_int(&t->tids_in_use, f->fs.type ? 2 : 1);
 313 }
 314
 315 static void
 316 remove_hftid(struct adapter *sc, struct filter_entry *f)
 317 {
 318 #ifdef INVARIANTS
 319         struct tid_info *t = &sc->tids;
 320
 321         mtx_assert(&t->hftid_lock, MA_OWNED);
 322 #endif
 323
 324         LIST_REMOVE(f, link_tid);
 325 }
 326
 327 /*
 328  * Input: driver's 32b filter mode.
 329  * Returns: hardware filter mode (bits to set in vlan_pri_map) for the input.
 330  */
 331 static uint16_t
 332 mode_to_fconf(uint32_t mode)
 333 {
 334         uint32_t fconf = 0;
 335
 336         if (mode & T4_FILTER_IP_FRAGMENT)
 337                 fconf |= F_FRAGMENTATION;
 338
 339         if (mode & T4_FILTER_MPS_HIT_TYPE)
 340                 fconf |= F_MPSHITTYPE;
 341
 342         if (mode & T4_FILTER_MAC_IDX)
 343                 fconf |= F_MACMATCH;
 344
 345         if (mode & T4_FILTER_ETH_TYPE)
 346                 fconf |= F_ETHERTYPE;
 347
 348         if (mode & T4_FILTER_IP_PROTO)
 349                 fconf |= F_PROTOCOL;
 350
 351         if (mode & T4_FILTER_IP_TOS)
 352                 fconf |= F_TOS;
 353
 354         if (mode & T4_FILTER_VLAN)
 355                 fconf |= F_VLAN;
 356
 357         if (mode & T4_FILTER_VNIC)
 358                 fconf |= F_VNIC_ID;
 359
 360         if (mode & T4_FILTER_PORT)
 361                 fconf |= F_PORT;
 362
 363         if (mode & T4_FILTER_FCoE)
 364                 fconf |= F_FCOE;
 365
 366         return (fconf);
 367 }
 368
 369 /*
 370  * Input: driver's 32b filter mode.
 371  * Returns: hardware vnic mode (ingress config) matching the input.
 372  */
 373 static int
 374 mode_to_iconf(uint32_t mode)
 375 {
 376         if ((mode & T4_FILTER_VNIC) == 0)
 377                 return (-1);    /* ingress config doesn't matter. */
 378
 379         if (mode & T4_FILTER_IC_VNIC)
 380                 return (FW_VNIC_MODE_PF_VF);
 381         else if (mode & T4_FILTER_IC_ENCAP)
 382                 return (FW_VNIC_MODE_ENCAP_EN);
 383         else
 384                 return (FW_VNIC_MODE_OUTER_VLAN);
 385 }
 386
 387 static int
 388 check_fspec_against_fconf_iconf(struct adapter *sc,
 389     struct t4_filter_specification *fs)
 390 {
 391         struct tp_params *tpp = &sc->params.tp;
 392         uint32_t fconf = 0;
 393
 394         if (fs->val.frag || fs->mask.frag)
 395                 fconf |= F_FRAGMENTATION;
 396
 397         if (fs->val.matchtype || fs->mask.matchtype)
 398                 fconf |= F_MPSHITTYPE;
 399
 400         if (fs->val.macidx || fs->mask.macidx)
 401                 fconf |= F_MACMATCH;
 402
 403         if (fs->val.ethtype || fs->mask.ethtype)
 404                 fconf |= F_ETHERTYPE;
 405
 406         if (fs->val.proto || fs->mask.proto)
 407                 fconf |= F_PROTOCOL;
 408
 409         if (fs->val.tos || fs->mask.tos)
 410                 fconf |= F_TOS;
 411
 412         if (fs->val.vlan_vld || fs->mask.vlan_vld)
 413                 fconf |= F_VLAN;
 414
 415         if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
 416                 if (tpp->vnic_mode != FW_VNIC_MODE_OUTER_VLAN)
 417                         return (EINVAL);
 418                 fconf |= F_VNIC_ID;
 419         }
 420
 421         if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
 422                 if (tpp->vnic_mode != FW_VNIC_MODE_PF_VF)
 423                         return (EINVAL);
 424                 fconf |= F_VNIC_ID;
 425         }
 426
 427 #ifdef notyet
 428         if (fs->val.encap_vld || fs->mask.encap_vld) {
 429                 if (tpp->vnic_mode != FW_VNIC_MODE_ENCAP_EN);
 430                         return (EINVAL);
 431                 fconf |= F_VNIC_ID;
 432         }
 433 #endif
 434
 435         if (fs->val.iport || fs->mask.iport)
 436                 fconf |= F_PORT;
 437
 438         if (fs->val.fcoe || fs->mask.fcoe)
 439                 fconf |= F_FCOE;
 440
 441         if ((tpp->filter_mode | fconf) != tpp->filter_mode)
 442                 return (E2BIG);
 443
 444         return (0);
 445 }
 446
 447 /*
 448  * Input: hardware filter configuration (filter mode/mask, ingress config).
 449  * Input: driver's 32b filter mode matching the input.
 450  */
 451 static uint32_t
 452 fconf_to_mode(uint16_t hwmode, int vnic_mode)
 453 {
 454         uint32_t mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
 455             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
 456
 457         if (hwmode & F_FRAGMENTATION)
 458                 mode |= T4_FILTER_IP_FRAGMENT;
 459         if (hwmode & F_MPSHITTYPE)
 460                 mode |= T4_FILTER_MPS_HIT_TYPE;
 461         if (hwmode & F_MACMATCH)
 462                 mode |= T4_FILTER_MAC_IDX;
 463         if (hwmode & F_ETHERTYPE)
 464                 mode |= T4_FILTER_ETH_TYPE;
 465         if (hwmode & F_PROTOCOL)
 466                 mode |= T4_FILTER_IP_PROTO;
 467         if (hwmode & F_TOS)
 468                 mode |= T4_FILTER_IP_TOS;
 469         if (hwmode & F_VLAN)
 470                 mode |= T4_FILTER_VLAN;
 471         if (hwmode & F_VNIC_ID)
 472                 mode |= T4_FILTER_VNIC; /* real meaning depends on vnic_mode. */
 473         if (hwmode & F_PORT)
 474                 mode |= T4_FILTER_PORT;
 475         if (hwmode & F_FCOE)
 476                 mode |= T4_FILTER_FCoE;
 477
 478         switch (vnic_mode) {
 479         case FW_VNIC_MODE_PF_VF:
 480                 mode |= T4_FILTER_IC_VNIC;
 481                 break;
 482         case FW_VNIC_MODE_ENCAP_EN:
 483                 mode |= T4_FILTER_IC_ENCAP;
 484                 break;
 485         case FW_VNIC_MODE_OUTER_VLAN:
 486         default:
 487                 break;
 488         }
 489
 490         return (mode);
 491 }
 492
 493 int
 494 get_filter_mode(struct adapter *sc, uint32_t *mode)
 495 {
 496         struct tp_params *tp = &sc->params.tp;
 497         uint16_t filter_mode;
 498
 499         /* Filter mask must comply with the global filter mode. */
 500         MPASS((tp->filter_mode | tp->filter_mask) == tp->filter_mode);
 501
 502         /* Non-zero incoming value in mode means "hashfilter mode". */
 503         filter_mode = *mode ? tp->filter_mask : tp->filter_mode;
 504         *mode = fconf_to_mode(filter_mode, tp->vnic_mode);
 505
 506         return (0);
 507 }
 508
 509 int
 510 set_filter_mode(struct adapter *sc, uint32_t mode)
 511 {
 512         struct tp_params *tp = &sc->params.tp;
 513         int rc, iconf;
 514         uint16_t fconf;
 515
 516         iconf = mode_to_iconf(mode);
 517         fconf = mode_to_fconf(mode);
 518         if ((iconf == -1 || iconf == tp->vnic_mode) && fconf == tp->filter_mode)
 519                 return (0);     /* Nothing to do */
 520
 521         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setfm");
 522         if (rc)
 523                 return (rc);
 524
 525         if (hw_off_limits(sc)) {
 526                 rc = ENXIO;
 527                 goto done;
 528         }
 529
 530         if (sc->tids.ftids_in_use > 0 ||        /* TCAM filters active */
 531             sc->tids.hpftids_in_use > 0 ||      /* hi-pri TCAM filters active */
 532             sc->tids.tids_in_use > 0) {         /* TOE or hashfilters active */
 533                 rc = EBUSY;
 534                 goto done;
 535         }
 536
 537 #ifdef TCP_OFFLOAD
 538         if (uld_active(sc, ULD_TOM)) {
 539                 rc = EBUSY;
 540                 goto done;
 541         }
 542 #endif
 543
 544         /* Note that filter mask will get clipped to the new filter mode. */
 545         rc = -t4_set_filter_cfg(sc, fconf, -1, iconf);
 546 done:
 547         end_synchronized_op(sc, 0);
 548         return (rc);
 549 }
 550
 551 int
 552 set_filter_mask(struct adapter *sc, uint32_t mode)
 553 {
 554         struct tp_params *tp = &sc->params.tp;
 555         int rc, iconf;
 556         uint16_t fmask;
 557
 558         iconf = mode_to_iconf(mode);
 559         fmask = mode_to_fconf(mode);
 560         if ((iconf == -1 || iconf == tp->vnic_mode) && fmask == tp->filter_mask)
 561                 return (0);     /* Nothing to do */
 562
 563         /*
 564          * We aren't going to change the global filter mode or VNIC mode here.
 565          * The given filter mask must conform to them.
 566          */
 567         if ((fmask | tp->filter_mode) != tp->filter_mode)
 568                 return (EINVAL);
 569         if (iconf != -1 && iconf != tp->vnic_mode)
 570                 return (EINVAL);
 571
 572         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sethfm");
 573         if (rc)
 574                 return (rc);
 575
 576         if (hw_off_limits(sc)) {
 577                 rc = ENXIO;
 578                 goto done;
 579         }
 580
 581         if (sc->tids.tids_in_use > 0) {         /* TOE or hashfilters active */
 582                 rc = EBUSY;
 583                 goto done;
 584         }
 585
 586 #ifdef TCP_OFFLOAD
 587         if (uld_active(sc, ULD_TOM)) {
 588                 rc = EBUSY;
 589                 goto done;
 590         }
 591 #endif
 592         rc = -t4_set_filter_cfg(sc, -1, fmask, -1);
 593 done:
 594         end_synchronized_op(sc, 0);
 595         return (rc);
 596 }
 597
 598 static inline uint64_t
 599 get_filter_hits(struct adapter *sc, uint32_t tid)
 600 {
 601         uint32_t tcb_addr;
 602         uint64_t hits;
 603
 604         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE;
 605
 606         mtx_lock(&sc->reg_lock);
 607         if (hw_off_limits(sc))
 608                 hits = 0;
 609         else if (is_t4(sc)) {
 610                 uint64_t t;
 611
 612                 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&t, 8);
 613                 hits = be64toh(t);
 614         } else {
 615                 uint32_t t;
 616
 617                 read_via_memwin(sc, 0, tcb_addr + 24, &t, 4);
 618                 hits = be32toh(t);
 619         }
 620         mtx_unlock(&sc->reg_lock);
 621
 622         return (hits);
 623 }
 624
 625 int
 626 get_filter(struct adapter *sc, struct t4_filter *t)
 627 {
 628         if (t->fs.hash)
 629                 return (get_hashfilter(sc, t));
 630         else
 631                 return (get_tcamfilter(sc, t));
 632 }
 633
 634 static int
 635 set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te,
 636     struct smt_entry *smt)
 637 {
 638         struct filter_entry *f;
 639         struct fw_filter2_wr *fwr;
 640         u_int vnic_vld, vnic_vld_mask;
 641         struct wrq_cookie cookie;
 642         int i, rc, busy, locked;
 643         u_int tid;
 644         const int ntids = t->fs.type ? 4 : 1;
 645
 646         MPASS(!t->fs.hash);
 647         /* Already validated against fconf, iconf */
 648         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
 649         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
 650
 651         if (separate_hpfilter_region(sc) && t->fs.prio) {
 652                 MPASS(t->idx < sc->tids.nhpftids);
 653                 f = &sc->tids.hpftid_tab[t->idx];
 654                 tid = sc->tids.hpftid_base + t->idx;
 655         } else {
 656                 MPASS(t->idx < sc->tids.nftids);
 657                 f = &sc->tids.ftid_tab[t->idx];
 658                 tid = sc->tids.ftid_base + t->idx;
 659         }
 660         rc = busy = locked = 0;
 661         mtx_lock(&sc->tids.ftid_lock);
 662         for (i = 0; i < ntids; i++) {
 663                 busy += f[i].pending + f[i].valid;
 664                 locked += f[i].locked;
 665         }
 666         if (locked > 0)
 667                 rc = EPERM;
 668         else if (busy > 0)
 669                 rc = EBUSY;
 670         else {
 671                 int len16;
 672
 673                 if (sc->params.filter2_wr_support)
 674                         len16 = howmany(sizeof(struct fw_filter2_wr), 16);
 675                 else
 676                         len16 = howmany(sizeof(struct fw_filter_wr), 16);
 677                 fwr = start_wrq_wr(&sc->sge.ctrlq[0], len16, &cookie);
 678                 if (__predict_false(fwr == NULL))
 679                         rc = ENOMEM;
 680                 else {
 681                         f->pending = 1;
 682                         if (separate_hpfilter_region(sc) && t->fs.prio)
 683                                 sc->tids.hpftids_in_use++;
 684                         else
 685                                 sc->tids.ftids_in_use++;
 686                 }
 687         }
 688         mtx_unlock(&sc->tids.ftid_lock);
 689         if (rc != 0)
 690                 return (rc);
 691
 692         /*
 693          * Can't fail now.  A set-filter WR will definitely be sent.
 694          */
 695
 696         f->tid = tid;
 697         f->fs = t->fs;
 698         f->l2te = l2te;
 699         f->smt = smt;
 700
 701         if (t->fs.val.pfvf_vld || t->fs.val.ovlan_vld)
 702                 vnic_vld = 1;
 703         else
 704                 vnic_vld = 0;
 705         if (t->fs.mask.pfvf_vld || t->fs.mask.ovlan_vld)
 706                 vnic_vld_mask = 1;
 707         else
 708                 vnic_vld_mask = 0;
 709
 710         bzero(fwr, sizeof(*fwr));
 711         if (sc->params.filter2_wr_support)
 712                 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER2_WR));
 713         else
 714                 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
 715         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
 716         fwr->tid_to_iq =
 717             htobe32(V_FW_FILTER_WR_TID(f->tid) |
 718                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
 719                 V_FW_FILTER_WR_NOREPLY(0) |
 720                 V_FW_FILTER_WR_IQ(f->fs.iq));
 721         fwr->del_filter_to_l2tix =
 722             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
 723                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
 724                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
 725                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
 726                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
 727                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
 728                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
 729                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
 730                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
 731                     f->fs.newvlan == VLAN_REWRITE) |
 732                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
 733                     f->fs.newvlan == VLAN_REWRITE) |
 734                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
 735                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
 736                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
 737                 V_FW_FILTER_WR_L2TIX(f->l2te ? f->l2te->idx : 0));
 738         fwr->ethtype = htobe16(f->fs.val.ethtype);
 739         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
 740         fwr->frag_to_ovlan_vldm =
 741             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
 742                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
 743                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
 744                 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
 745                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
 746                 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
 747         fwr->smac_sel = 0;
 748         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
 749             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
 750         fwr->maci_to_matchtypem =
 751             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
 752                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
 753                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
 754                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
 755                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
 756                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
 757                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
 758                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
 759         fwr->ptcl = f->fs.val.proto;
 760         fwr->ptclm = f->fs.mask.proto;
 761         fwr->ttyp = f->fs.val.tos;
 762         fwr->ttypm = f->fs.mask.tos;
 763         fwr->ivlan = htobe16(f->fs.val.vlan);
 764         fwr->ivlanm = htobe16(f->fs.mask.vlan);
 765         fwr->ovlan = htobe16(f->fs.val.vnic);
 766         fwr->ovlanm = htobe16(f->fs.mask.vnic);
 767         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
 768         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
 769         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
 770         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
 771         fwr->lp = htobe16(f->fs.val.dport);
 772         fwr->lpm = htobe16(f->fs.mask.dport);
 773         fwr->fp = htobe16(f->fs.val.sport);
 774         fwr->fpm = htobe16(f->fs.mask.sport);
 775         /* sma = 0 tells the fw to use SMAC_SEL for source MAC address */
 776         bzero(fwr->sma, sizeof (fwr->sma));
 777         if (sc->params.filter2_wr_support) {
 778                 fwr->filter_type_swapmac =
 779                     V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac);
 780                 fwr->natmode_to_ulp_type =
 781                     V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ?
 782                         ULP_MODE_TCPDDP : ULP_MODE_NONE) |
 783                     V_FW_FILTER2_WR_NATFLAGCHECK(f->fs.nat_flag_chk) |
 784                     V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode);
 785                 memcpy(fwr->newlip, f->fs.nat_dip, sizeof(fwr->newlip));
 786                 memcpy(fwr->newfip, f->fs.nat_sip, sizeof(fwr->newfip));
 787                 fwr->newlport = htobe16(f->fs.nat_dport);
 788                 fwr->newfport = htobe16(f->fs.nat_sport);
 789                 fwr->natseqcheck = htobe32(f->fs.nat_seq_chk);
 790         }
 791         commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie);
 792
 793         /* Wait for response. */
 794         mtx_lock(&sc->tids.ftid_lock);
 795         for (;;) {
 796                 if (f->pending == 0) {
 797                         rc = f->valid ? 0 : EIO;
 798                         break;
 799                 }
 800                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
 801                         rc = EINPROGRESS;
 802                         break;
 803                 }
 804         }
 805         mtx_unlock(&sc->tids.ftid_lock);
 806         return (rc);
 807 }
 808
 809 static int
 810 hashfilter_ntuple(struct adapter *sc, const struct t4_filter_specification *fs,
 811     uint64_t *ftuple)
 812 {
 813         struct tp_params *tp = &sc->params.tp;
 814         uint16_t fmask;
 815
 816         *ftuple = fmask = 0;
 817
 818         /*
 819          * Initialize each of the fields which we care about which are present
 820          * in the Compressed Filter Tuple.
 821          */
 822         if (tp->vlan_shift >= 0 && fs->mask.vlan) {
 823                 *ftuple |= (uint64_t)(F_FT_VLAN_VLD | fs->val.vlan) <<
 824                     tp->vlan_shift;
 825                 fmask |= F_VLAN;
 826         }
 827
 828         if (tp->port_shift >= 0 && fs->mask.iport) {
 829                 *ftuple |= (uint64_t)fs->val.iport << tp->port_shift;
 830                 fmask |= F_PORT;
 831         }
 832
 833         if (tp->protocol_shift >= 0 && fs->mask.proto) {
 834                 *ftuple |= (uint64_t)fs->val.proto << tp->protocol_shift;
 835                 fmask |= F_PROTOCOL;
 836         }
 837
 838         if (tp->tos_shift >= 0 && fs->mask.tos) {
 839                 *ftuple |= (uint64_t)(fs->val.tos) << tp->tos_shift;
 840                 fmask |= F_TOS;
 841         }
 842
 843         if (tp->vnic_shift >= 0 && fs->mask.vnic) {
 844                 /* vnic_mode was already validated. */
 845                 if (tp->vnic_mode == FW_VNIC_MODE_PF_VF)
 846                         MPASS(fs->mask.pfvf_vld);
 847                 else if (tp->vnic_mode == FW_VNIC_MODE_OUTER_VLAN)
 848                         MPASS(fs->mask.ovlan_vld);
 849 #ifdef notyet
 850                 else if (tp->vnic_mode == FW_VNIC_MODE_ENCAP_EN)
 851                         MPASS(fs->mask.encap_vld);
 852 #endif
 853                 *ftuple |= ((1ULL << 16) | fs->val.vnic) << tp->vnic_shift;
 854                 fmask |= F_VNIC_ID;
 855         }
 856
 857         if (tp->macmatch_shift >= 0 && fs->mask.macidx) {
 858                 *ftuple |= (uint64_t)(fs->val.macidx) << tp->macmatch_shift;
 859                 fmask |= F_MACMATCH;
 860         }
 861
 862         if (tp->ethertype_shift >= 0 && fs->mask.ethtype) {
 863                 *ftuple |= (uint64_t)(fs->val.ethtype) << tp->ethertype_shift;
 864                 fmask |= F_ETHERTYPE;
 865         }
 866
 867         if (tp->matchtype_shift >= 0 && fs->mask.matchtype) {
 868                 *ftuple |= (uint64_t)(fs->val.matchtype) << tp->matchtype_shift;
 869                 fmask |= F_MPSHITTYPE;
 870         }
 871
 872         if (tp->frag_shift >= 0 && fs->mask.frag) {
 873                 *ftuple |= (uint64_t)(fs->val.frag) << tp->frag_shift;
 874                 fmask |= F_FRAGMENTATION;
 875         }
 876
 877         if (tp->fcoe_shift >= 0 && fs->mask.fcoe) {
 878                 *ftuple |= (uint64_t)(fs->val.fcoe) << tp->fcoe_shift;
 879                 fmask |= F_FCOE;
 880         }
 881
 882         /* A hashfilter must conform to the hardware filter mask. */
 883         if (fmask != tp->filter_mask)
 884                 return (EINVAL);
 885
 886         return (0);
 887 }
 888
 889 static bool
 890 is_4tuple_specified(struct t4_filter_specification *fs)
 891 {
 892         int i;
 893         const int n = fs->type ? 16 : 4;
 894
 895         if (fs->mask.sport != 0xffff || fs->mask.dport != 0xffff)
 896                 return (false);
 897
 898         for (i = 0; i < n; i++) {
 899                 if (fs->mask.sip[i] != 0xff)
 900                         return (false);
 901                 if (fs->mask.dip[i] != 0xff)
 902                         return (false);
 903         }
 904
 905         return (true);
 906 }
 907
 908 int
 909 set_filter(struct adapter *sc, struct t4_filter *t)
 910 {
 911         struct tid_info *ti = &sc->tids;
 912         struct l2t_entry *l2te = NULL;
 913         struct smt_entry *smt = NULL;
 914         uint64_t ftuple;
 915         int rc;
 916
 917         /*
 918          * Basic filter checks first.
 919          */
 920
 921         if (t->fs.hash) {
 922                 if (!is_hashfilter(sc) || ti->ntids == 0)
 923                         return (ENOTSUP);
 924                 /* Hardware, not user, selects a tid for hashfilters. */
 925                 if (t->idx != (uint32_t)-1)
 926                         return (EINVAL);
 927                 /* T5 can't count hashfilter hits. */
 928                 if (is_t5(sc) && t->fs.hitcnts)
 929                         return (EINVAL);
 930                 if (!is_4tuple_specified(&t->fs))
 931                         return (EINVAL);
 932                 rc = hashfilter_ntuple(sc, &t->fs, &ftuple);
 933                 if (rc != 0)
 934                         return (rc);
 935         } else {
 936                 if (separate_hpfilter_region(sc) && t->fs.prio) {
 937                         if (ti->nhpftids == 0)
 938                                 return (ENOTSUP);
 939                         if (t->idx >= ti->nhpftids)
 940                                 return (EINVAL);
 941                 } else {
 942                         if (ti->nftids == 0)
 943                                 return (ENOTSUP);
 944                         if (t->idx >= ti->nftids)
 945                                 return (EINVAL);
 946                 }
 947                 /* IPv6 filter idx must be 4 aligned */
 948                 if (t->fs.type == 1 &&
 949                     ((t->idx & 0x3) || t->idx + 4 >= ti->nftids))
 950                         return (EINVAL);
 951         }
 952
 953         /* T4 doesn't support VLAN tag removal or rewrite, swapmac, and NAT. */
 954         if (is_t4(sc) && t->fs.action == FILTER_SWITCH &&
 955             (t->fs.newvlan == VLAN_REMOVE || t->fs.newvlan == VLAN_REWRITE ||
 956             t->fs.swapmac || t->fs.nat_mode))
 957                 return (ENOTSUP);
 958
 959         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= sc->params.nports)
 960                 return (EINVAL);
 961         if (t->fs.val.iport >= sc->params.nports)
 962                 return (EINVAL);
 963
 964         /* Can't specify an iqid/rss_info if not steering. */
 965         if (!t->fs.dirsteer && !t->fs.dirsteerhash && !t->fs.maskhash && t->fs.iq)
 966                 return (EINVAL);
 967
 968         /* Validate against the global filter mode and ingress config */
 969         rc = check_fspec_against_fconf_iconf(sc, &t->fs);
 970         if (rc != 0)
 971                 return (rc);
 972
 973         /*
 974          * Basic checks passed.  Make sure the queues and tid tables are setup.
 975          */
 976
 977         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
 978         if (rc)
 979                 return (rc);
 980
 981         if (hw_off_limits(sc)) {
 982                 rc = ENXIO;
 983                 goto done;
 984         }
 985
 986         if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_init(sc)) != 0))
 987                 goto done;
 988
 989         if (t->fs.hash) {
 990                 if (__predict_false(ti->hftid_hash_4t == NULL)) {
 991                         rc = alloc_hftid_hash(&sc->tids, HASH_NOWAIT);
 992                         if (rc != 0)
 993                                 goto done;
 994                 }
 995         } else if (separate_hpfilter_region(sc) && t->fs.prio &&
 996             __predict_false(ti->hpftid_tab == NULL)) {
 997                 MPASS(ti->nhpftids != 0);
 998                 KASSERT(ti->hpftids_in_use == 0,
 999                     ("%s: no memory allocated but hpftids_in_use is %u",
1000                     __func__, ti->hpftids_in_use));
1001                 ti->hpftid_tab = malloc(sizeof(struct filter_entry) *
1002                     ti->nhpftids, M_CXGBE, M_NOWAIT | M_ZERO);
1003                 if (ti->hpftid_tab == NULL) {
1004                         rc = ENOMEM;
1005                         goto done;
1006                 }
1007                 if (!mtx_initialized(&sc->tids.ftid_lock)) {
1008                         mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
1009                         cv_init(&ti->ftid_cv, "t4fcv");
1010                 }
1011         } else if (__predict_false(ti->ftid_tab == NULL)) {
1012                 MPASS(ti->nftids != 0);
1013                 KASSERT(ti->ftids_in_use == 0,
1014                     ("%s: no memory allocated but ftids_in_use is %u",
1015                     __func__, ti->ftids_in_use));
1016                 ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids,
1017                     M_CXGBE, M_NOWAIT | M_ZERO);
1018                 if (ti->ftid_tab == NULL) {
1019                         rc = ENOMEM;
1020                         goto done;
1021                 }
1022                 if (!mtx_initialized(&sc->tids.ftid_lock)) {
1023                         mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
1024                         cv_init(&ti->ftid_cv, "t4fcv");
1025                 }
1026         }
1027 done:
1028         end_synchronized_op(sc, 0);
1029         if (rc != 0)
1030                 return (rc);
1031
1032         /*
1033          * Allocate L2T entry, SMT entry, etc.
1034          */
1035
1036         if (t->fs.newdmac || t->fs.newvlan) {
1037                 /* This filter needs an L2T entry; allocate one. */
1038                 l2te = t4_l2t_alloc_switching(sc, t->fs.vlan, t->fs.eport,
1039                     t->fs.dmac);
1040                 if (__predict_false(l2te == NULL)) {
1041                         rc = EAGAIN;
1042                         goto error;
1043                 }
1044         }
1045
1046         if (t->fs.newsmac) {
1047                 /* This filter needs an SMT entry; allocate one. */
1048                 smt = t4_smt_alloc_switching(sc->smt, t->fs.smac);
1049                 if (__predict_false(smt == NULL)) {
1050                         rc = EAGAIN;
1051                         goto error;
1052                 }
1053                 rc = t4_smt_set_switching(sc, smt, 0x0, t->fs.smac);
1054                 if (rc)
1055                         goto error;
1056         }
1057
1058         if (t->fs.hash)
1059                 rc = set_hashfilter(sc, t, ftuple, l2te, smt);
1060         else
1061                 rc = set_tcamfilter(sc, t, l2te, smt);
1062
1063         if (rc != 0 && rc != EINPROGRESS) {
1064 error:
1065                 if (l2te)
1066                         t4_l2t_release(l2te);
1067                 if (smt)
1068                         t4_smt_release(smt);
1069         }
1070         return (rc);
1071 }
1072
1073 static int
1074 del_tcamfilter(struct adapter *sc, struct t4_filter *t)
1075 {
1076         struct filter_entry *f;
1077         struct fw_filter_wr *fwr;
1078         struct wrq_cookie cookie;
1079         int rc, nfilters;
1080 #ifdef INVARIANTS
1081         u_int tid_base;
1082 #endif
1083
1084         mtx_lock(&sc->tids.ftid_lock);
1085         if (separate_hpfilter_region(sc) && t->fs.prio) {
1086                 nfilters = sc->tids.nhpftids;
1087                 f = sc->tids.hpftid_tab;
1088 #ifdef INVARIANTS
1089                 tid_base = sc->tids.hpftid_base;
1090 #endif
1091         } else {
1092                 nfilters = sc->tids.nftids;
1093                 f = sc->tids.ftid_tab;
1094 #ifdef INVARIANTS
1095                 tid_base = sc->tids.ftid_base;
1096 #endif
1097         }
1098         MPASS(f != NULL);       /* Caller checked this. */
1099         if (t->idx >= nfilters) {
1100                 rc = EINVAL;
1101                 goto done;
1102         }
1103         f += t->idx;
1104
1105         if (f->locked) {
1106                 rc = EPERM;
1107                 goto done;
1108         }
1109         if (f->pending) {
1110                 rc = EBUSY;
1111                 goto done;
1112         }
1113         if (f->valid == 0) {
1114                 rc = EINVAL;
1115                 goto done;
1116         }
1117         MPASS(f->tid == tid_base + t->idx);
1118         fwr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*fwr), 16), &cookie);
1119         if (fwr == NULL) {
1120                 rc = ENOMEM;
1121                 goto done;
1122         }
1123
1124         bzero(fwr, sizeof (*fwr));
1125         t4_mk_filtdelwr(f->tid, fwr, sc->sge.fwq.abs_id);
1126         f->pending = 1;
1127         commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie);
1128         t->fs = f->fs;  /* extra info for the caller */
1129
1130         for (;;) {
1131                 if (f->pending == 0) {
1132                         rc = f->valid ? EIO : 0;
1133                         break;
1134                 }
1135                 if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
1136                         rc = EINPROGRESS;
1137                         break;
1138                 }
1139         }
1140 done:
1141         mtx_unlock(&sc->tids.ftid_lock);
1142         return (rc);
1143 }
1144
1145 int
1146 del_filter(struct adapter *sc, struct t4_filter *t)
1147 {
1148
1149         /* No filters possible if not initialized yet. */
1150         if (!(sc->flags & FULL_INIT_DONE))
1151                 return (EINVAL);
1152
1153         /*
1154          * The checks for tid tables ensure that the locks that del_* will reach
1155          * for are initialized.
1156          */
1157         if (t->fs.hash) {
1158                 if (sc->tids.hftid_hash_4t != NULL)
1159                         return (del_hashfilter(sc, t));
1160         } else if (separate_hpfilter_region(sc) && t->fs.prio) {
1161                 if (sc->tids.hpftid_tab != NULL)
1162                         return (del_tcamfilter(sc, t));
1163         } else {
1164                 if (sc->tids.ftid_tab != NULL)
1165                         return (del_tcamfilter(sc, t));
1166         }
1167
1168         return (EINVAL);
1169 }
1170
1171 /*
1172  * Release secondary resources associated with the filter.
1173  */
1174 static void
1175 free_filter_resources(struct filter_entry *f)
1176 {
1177
1178         if (f->l2te) {
1179                 t4_l2t_release(f->l2te);
1180                 f->l2te = NULL;
1181         }
1182         if (f->smt) {
1183                 t4_smt_release(f->smt);
1184                 f->smt = NULL;
1185         }
1186 }
1187
1188 static int
1189 set_tcb_field(struct adapter *sc, u_int tid, uint16_t word, uint64_t mask,
1190     uint64_t val, int no_reply)
1191 {
1192         struct wrq_cookie cookie;
1193         struct cpl_set_tcb_field *req;
1194
1195         req = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*req), 16), &cookie);
1196         if (req == NULL)
1197                 return (ENOMEM);
1198         bzero(req, sizeof(*req));
1199         INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid);
1200         if (no_reply == 0) {
1201                 req->reply_ctrl = htobe16(V_QUEUENO(sc->sge.fwq.abs_id) |
1202                     V_NO_REPLY(0));
1203         } else
1204                 req->reply_ctrl = htobe16(V_NO_REPLY(1));
1205         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(CPL_COOKIE_HASHFILTER));
1206         req->mask = htobe64(mask);
1207         req->val = htobe64(val);
1208         commit_wrq_wr(&sc->sge.ctrlq[0], req, &cookie);
1209
1210         return (0);
1211 }
1212
1213 /* Set one of the t_flags bits in the TCB. */
1214 static inline int
1215 set_tcb_tflag(struct adapter *sc, int tid, u_int bit_pos, u_int val,
1216     u_int no_reply)
1217 {
1218
1219         return (set_tcb_field(sc, tid,  W_TCB_T_FLAGS, 1ULL << bit_pos,
1220             (uint64_t)val << bit_pos, no_reply));
1221 }
1222
1223 int
1224 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1225 {
1226         struct adapter *sc = iq->adapter;
1227         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
1228         u_int tid = GET_TID(rpl);
1229         u_int rc, idx;
1230         struct filter_entry *f;
1231
1232         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
1233             rss->opcode));
1234
1235
1236         if (is_hpftid(sc, tid)) {
1237                 idx = tid - sc->tids.hpftid_base;
1238                 f = &sc->tids.hpftid_tab[idx];
1239         } else if (is_ftid(sc, tid)) {
1240                 idx = tid - sc->tids.ftid_base;
1241                 f = &sc->tids.ftid_tab[idx];
1242         } else
1243                 panic("%s: FW reply for invalid TID %d.", __func__, tid);
1244
1245         MPASS(f->tid == tid);
1246         rc = G_COOKIE(rpl->cookie);
1247
1248         mtx_lock(&sc->tids.ftid_lock);
1249         KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.",
1250             __func__, rc, tid));
1251         switch(rc) {
1252         case FW_FILTER_WR_FLT_ADDED:
1253                 /* set-filter succeeded */
1254                 f->valid = 1;
1255                 if (f->fs.newsmac) {
1256                         MPASS(f->smt != NULL);
1257                         set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
1258                         set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
1259                             V_TCB_SMAC_SEL(M_TCB_SMAC_SEL),
1260                             V_TCB_SMAC_SEL(f->smt->idx), 1);
1261                         /* XXX: wait for reply to TCB update before !pending */
1262                 }
1263                 break;
1264         case FW_FILTER_WR_FLT_DELETED:
1265                 /* del-filter succeeded */
1266                 MPASS(f->valid == 1);
1267                 f->valid = 0;
1268                 /* Fall through */
1269         case FW_FILTER_WR_SMT_TBL_FULL:
1270                 /* set-filter failed due to lack of SMT space. */
1271                 MPASS(f->valid == 0);
1272                 free_filter_resources(f);
1273                 if (separate_hpfilter_region(sc) && f->fs.prio)
1274                         sc->tids.hpftids_in_use--;
1275                 else
1276                         sc->tids.ftids_in_use--;
1277                 break;
1278         case FW_FILTER_WR_SUCCESS:
1279         case FW_FILTER_WR_EINVAL:
1280         default:
1281                 panic("%s: unexpected reply %d for filter[%d].", __func__, rc,
1282                     idx);
1283         }
1284         f->pending = 0;
1285         cv_broadcast(&sc->tids.ftid_cv);
1286         mtx_unlock(&sc->tids.ftid_lock);
1287
1288         return (0);
1289 }
1290
1291 /*
1292  * This is the reply to the Active Open that created the filter.  Additional TCB
1293  * updates may be required to complete the filter configuration.
1294  */
1295 int
1296 t4_hashfilter_ao_rpl(struct sge_iq *iq, const struct rss_header *rss,
1297     struct mbuf *m)
1298 {
1299         struct adapter *sc = iq->adapter;
1300         const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
1301         u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
1302         u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
1303         struct filter_entry *f = lookup_atid(sc, atid);
1304
1305         KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1306
1307         mtx_lock(&sc->tids.hftid_lock);
1308         KASSERT(f->pending, ("%s: hashfilter[%p] isn't pending.", __func__, f));
1309         KASSERT(f->tid == -1, ("%s: hashfilter[%p] has tid %d already.",
1310             __func__, f, f->tid));
1311         if (status == CPL_ERR_NONE) {
1312                 f->tid = GET_TID(cpl);
1313                 MPASS(lookup_hftid(sc, f->tid) == NULL);
1314                 insert_hftid(sc, f);
1315                 /*
1316                  * Leave the filter pending until it is fully set up, which will
1317                  * be indicated by the reply to the last TCB update.  No need to
1318                  * unblock the ioctl thread either.
1319                  */
1320                 if (configure_hashfilter_tcb(sc, f) == EINPROGRESS)
1321                         goto done;
1322                 f->valid = 1;
1323                 f->pending = 0;
1324         } else {
1325                 /* provide errno instead of tid to ioctl */
1326                 f->tid = act_open_rpl_status_to_errno(status);
1327                 f->valid = 0;
1328                 f->pending = 0;
1329                 if (act_open_has_tid(status))
1330                         release_tid(sc, GET_TID(cpl), &sc->sge.ctrlq[0]);
1331                 free_filter_resources(f);
1332                 remove_hf(sc, f);
1333                 if (f->locked == 0)
1334                         free(f, M_CXGBE);
1335         }
1336         cv_broadcast(&sc->tids.hftid_cv);
1337 done:
1338         mtx_unlock(&sc->tids.hftid_lock);
1339
1340         free_atid(sc, atid);
1341         return (0);
1342 }
1343
1344 int
1345 t4_hashfilter_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss,
1346     struct mbuf *m)
1347 {
1348         struct adapter *sc = iq->adapter;
1349         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
1350         u_int tid = GET_TID(rpl);
1351         struct filter_entry *f;
1352
1353         mtx_lock(&sc->tids.hftid_lock);
1354         f = lookup_hftid(sc, tid);
1355         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
1356         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
1357             f, tid));
1358         KASSERT(f->valid == 0, ("%s: hashfilter %p [%u] is valid already.",
1359             __func__, f, tid));
1360         f->pending = 0;
1361         if (rpl->status == 0) {
1362                 f->valid = 1;
1363         } else {
1364                 f->tid = EIO;
1365                 f->valid = 0;
1366                 free_filter_resources(f);
1367                 remove_hftid(sc, f);
1368                 remove_hf(sc, f);
1369                 release_tid(sc, tid, &sc->sge.ctrlq[0]);
1370                 if (f->locked == 0)
1371                         free(f, M_CXGBE);
1372         }
1373         cv_broadcast(&sc->tids.hftid_cv);
1374         mtx_unlock(&sc->tids.hftid_lock);
1375
1376         return (0);
1377 }
1378
1379 int
1380 t4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss,
1381     struct mbuf *m)
1382 {
1383         struct adapter *sc = iq->adapter;
1384         const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1385         unsigned int tid = GET_TID(cpl);
1386         struct filter_entry *f;
1387
1388         mtx_lock(&sc->tids.hftid_lock);
1389         f = lookup_hftid(sc, tid);
1390         KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
1391         KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
1392             f, tid));
1393         KASSERT(f->valid, ("%s: hashfilter %p [%u] isn't valid.", __func__, f,
1394             tid));
1395         f->pending = 0;
1396         if (cpl->status == 0) {
1397                 f->valid = 0;
1398                 free_filter_resources(f);
1399                 remove_hftid(sc, f);
1400                 remove_hf(sc, f);
1401                 release_tid(sc, tid, &sc->sge.ctrlq[0]);
1402                 if (f->locked == 0)
1403                         free(f, M_CXGBE);
1404         }
1405         cv_broadcast(&sc->tids.hftid_cv);
1406         mtx_unlock(&sc->tids.hftid_lock);
1407
1408         return (0);
1409 }
1410
1411 static int
1412 get_tcamfilter(struct adapter *sc, struct t4_filter *t)
1413 {
1414         int i, nfilters;
1415         struct filter_entry *f;
1416         u_int in_use;
1417 #ifdef INVARIANTS
1418         u_int tid_base;
1419 #endif
1420
1421         MPASS(!t->fs.hash);
1422
1423         if (separate_hpfilter_region(sc) && t->fs.prio) {
1424                 nfilters = sc->tids.nhpftids;
1425                 f = sc->tids.hpftid_tab;
1426                 in_use = sc->tids.hpftids_in_use;
1427 #ifdef INVARIANTS
1428                 tid_base = sc->tids.hpftid_base;
1429 #endif
1430         } else {
1431                 nfilters = sc->tids.nftids;
1432                 f = sc->tids.ftid_tab;
1433                 in_use = sc->tids.ftids_in_use;
1434 #ifdef INVARIANTS
1435                 tid_base = sc->tids.ftid_base;
1436 #endif
1437         }
1438
1439         if (in_use == 0 || f == NULL || t->idx >= nfilters) {
1440                 t->idx = 0xffffffff;
1441                 return (0);
1442         }
1443
1444         f += t->idx;
1445         mtx_lock(&sc->tids.ftid_lock);
1446         for (i = t->idx; i < nfilters; i++, f++) {
1447                 if (f->valid) {
1448                         MPASS(f->tid == tid_base + i);
1449                         t->idx = i;
1450                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
1451                         t->smtidx = f->smt ? f->smt->idx : 0;
1452                         if (f->fs.hitcnts)
1453                                 t->hits = get_filter_hits(sc, f->tid);
1454                         else
1455                                 t->hits = UINT64_MAX;
1456                         t->fs = f->fs;
1457
1458                         goto done;
1459                 }
1460         }
1461         t->idx = 0xffffffff;
1462 done:
1463         mtx_unlock(&sc->tids.ftid_lock);
1464         return (0);
1465 }
1466
1467 static int
1468 get_hashfilter(struct adapter *sc, struct t4_filter *t)
1469 {
1470         struct tid_info *ti = &sc->tids;
1471         int tid;
1472         struct filter_entry *f;
1473         const int inv_tid = ti->ntids + ti->tid_base;
1474
1475         MPASS(t->fs.hash);
1476
1477         if (ti->tids_in_use == 0 || ti->hftid_hash_tid == NULL ||
1478             t->idx >= inv_tid) {
1479                 t->idx = 0xffffffff;
1480                 return (0);
1481         }
1482         if (t->idx < ti->tid_base)
1483                 t->idx = ti->tid_base;
1484
1485         mtx_lock(&ti->hftid_lock);
1486         for (tid = t->idx; tid < inv_tid; tid++) {
1487                 f = lookup_hftid(sc, tid);
1488                 if (f != NULL && f->valid) {
1489                         t->idx = tid;
1490                         t->l2tidx = f->l2te ? f->l2te->idx : 0;
1491                         t->smtidx = f->smt ? f->smt->idx : 0;
1492                         if (f->fs.hitcnts)
1493                                 t->hits = get_filter_hits(sc, tid);
1494                         else
1495                                 t->hits = UINT64_MAX;
1496                         t->fs = f->fs;
1497
1498                         goto done;
1499                 }
1500         }
1501         t->idx = 0xffffffff;
1502 done:
1503         mtx_unlock(&ti->hftid_lock);
1504         return (0);
1505 }
1506
1507 static void
1508 mk_act_open_req6(struct adapter *sc, struct filter_entry *f, int atid,
1509     uint64_t ftuple, struct cpl_act_open_req6 *cpl)
1510 {
1511         struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
1512         struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
1513
1514         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1515         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1516         MPASS(atid >= 0);
1517
1518         if (chip_id(sc) == CHELSIO_T5) {
1519                 INIT_TP_WR(cpl5, 0);
1520         } else {
1521                 INIT_TP_WR(cpl6, 0);
1522                 cpl6->rsvd2 = 0;
1523                 cpl6->opt3 = 0;
1524         }
1525
1526         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
1527             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1528             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1529         cpl->local_port = htobe16(f->fs.val.dport);
1530         cpl->peer_port = htobe16(f->fs.val.sport);
1531         cpl->local_ip_hi = *(uint64_t *)(&f->fs.val.dip);
1532         cpl->local_ip_lo = *(((uint64_t *)&f->fs.val.dip) + 1);
1533         cpl->peer_ip_hi = *(uint64_t *)(&f->fs.val.sip);
1534         cpl->peer_ip_lo = *(((uint64_t *)&f->fs.val.sip) + 1);
1535         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1536             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1537             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1538             V_NO_CONG(f->fs.rpttid) |
1539             V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1540             F_TCAM_BYPASS | F_NON_OFFLOAD);
1541
1542         cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1543         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1544             V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1545             V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1546             F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1547             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1548             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1549 }
1550
1551 static void
1552 mk_act_open_req(struct adapter *sc, struct filter_entry *f, int atid,
1553     uint64_t ftuple, struct cpl_act_open_req *cpl)
1554 {
1555         struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
1556         struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
1557
1558         /* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1559         MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1560         MPASS(atid >= 0);
1561
1562         if (chip_id(sc) == CHELSIO_T5) {
1563                 INIT_TP_WR(cpl5, 0);
1564         } else {
1565                 INIT_TP_WR(cpl6, 0);
1566                 cpl6->rsvd2 = 0;
1567                 cpl6->opt3 = 0;
1568         }
1569
1570         OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
1571             V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1572             V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1573         cpl->local_port = htobe16(f->fs.val.dport);
1574         cpl->peer_port = htobe16(f->fs.val.sport);
1575         cpl->local_ip = f->fs.val.dip[0] | f->fs.val.dip[1] << 8 |
1576             f->fs.val.dip[2] << 16 | f->fs.val.dip[3] << 24;
1577         cpl->peer_ip = f->fs.val.sip[0] | f->fs.val.sip[1] << 8 |
1578                 f->fs.val.sip[2] << 16 | f->fs.val.sip[3] << 24;
1579         cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1580             f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1581             V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1582             V_NO_CONG(f->fs.rpttid) |
1583             V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1584             F_TCAM_BYPASS | F_NON_OFFLOAD);
1585
1586         cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1587         cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1588             V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1589             V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1590             F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1591             V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1592             V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1593 }
1594
1595 static int
1596 act_open_cpl_len16(struct adapter *sc, int isipv6)
1597 {
1598         int idx;
1599         static const int sz_table[3][2] = {
1600                 {
1601                         howmany(sizeof (struct cpl_act_open_req), 16),
1602                         howmany(sizeof (struct cpl_act_open_req6), 16)
1603                 },
1604                 {
1605                         howmany(sizeof (struct cpl_t5_act_open_req), 16),
1606                         howmany(sizeof (struct cpl_t5_act_open_req6), 16)
1607                 },
1608                 {
1609                         howmany(sizeof (struct cpl_t6_act_open_req), 16),
1610                         howmany(sizeof (struct cpl_t6_act_open_req6), 16)
1611                 },
1612         };
1613
1614         MPASS(chip_id(sc) >= CHELSIO_T4);
1615         idx = min(chip_id(sc) - CHELSIO_T4, 2);
1616
1617         return (sz_table[idx][!!isipv6]);
1618 }
1619
1620 static int
1621 set_hashfilter(struct adapter *sc, struct t4_filter *t, uint64_t ftuple,
1622     struct l2t_entry *l2te, struct smt_entry *smt)
1623 {
1624         void *wr;
1625         struct wrq_cookie cookie;
1626         struct filter_entry *f;
1627         int rc, atid = -1;
1628         uint32_t hash;
1629
1630         MPASS(t->fs.hash);
1631         /* Already validated against fconf, iconf */
1632         MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
1633         MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
1634
1635         hash = hf_hashfn_4t(&t->fs);
1636
1637         mtx_lock(&sc->tids.hftid_lock);
1638         if (lookup_hf(sc, &t->fs, hash) != NULL) {
1639                 rc = EEXIST;
1640                 goto done;
1641         }
1642
1643         f = malloc(sizeof(*f), M_CXGBE, M_ZERO | M_NOWAIT);
1644         if (__predict_false(f == NULL)) {
1645                 rc = ENOMEM;
1646                 goto done;
1647         }
1648         f->fs = t->fs;
1649         f->l2te = l2te;
1650         f->smt = smt;
1651
1652         atid = alloc_atid(sc, f);
1653         if (__predict_false(atid) == -1) {
1654                 free(f, M_CXGBE);
1655                 rc = EAGAIN;
1656                 goto done;
1657         }
1658         MPASS(atid >= 0);
1659
1660         wr = start_wrq_wr(&sc->sge.ctrlq[0], act_open_cpl_len16(sc, f->fs.type),
1661             &cookie);
1662         if (wr == NULL) {
1663                 free_atid(sc, atid);
1664                 free(f, M_CXGBE);
1665                 rc = ENOMEM;
1666                 goto done;
1667         }
1668         if (f->fs.type)
1669                 mk_act_open_req6(sc, f, atid, ftuple, wr);
1670         else
1671                 mk_act_open_req(sc, f, atid, ftuple, wr);
1672
1673         f->locked = 1; /* ithread mustn't free f if ioctl is still around. */
1674         f->pending = 1;
1675         f->tid = -1;
1676         insert_hf(sc, f, hash);
1677         commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie);
1678
1679         for (;;) {
1680                 MPASS(f->locked);
1681                 if (f->pending == 0) {
1682                         if (f->valid) {
1683                                 rc = 0;
1684                                 f->locked = 0;
1685                                 t->idx = f->tid;
1686                         } else {
1687                                 rc = f->tid;
1688                                 free(f, M_CXGBE);
1689                         }
1690                         break;
1691                 }
1692                 if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1693                         f->locked = 0;
1694                         rc = EINPROGRESS;
1695                         break;
1696                 }
1697         }
1698 done:
1699         mtx_unlock(&sc->tids.hftid_lock);
1700         return (rc);
1701 }
1702
1703 /* SET_TCB_FIELD sent as a ULP command looks like this */
1704 #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \
1705     sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core))
1706
1707 static void *
1708 mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, uint64_t word, uint64_t mask,
1709                 uint64_t val, uint32_t tid, uint32_t qid)
1710 {
1711         struct ulptx_idata *ulpsc;
1712         struct cpl_set_tcb_field_core *req;
1713
1714         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1715         ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
1716
1717         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1718         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1719         ulpsc->len = htobe32(sizeof(*req));
1720
1721         req = (struct cpl_set_tcb_field_core *)(ulpsc + 1);
1722         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1723         req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(qid));
1724         req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0));
1725         req->mask = htobe64(mask);
1726         req->val = htobe64(val);
1727
1728         ulpsc = (struct ulptx_idata *)(req + 1);
1729         if (LEN__SET_TCB_FIELD_ULP % 16) {
1730                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1731                 ulpsc->len = htobe32(0);
1732                 return (ulpsc + 1);
1733         }
1734         return (ulpsc);
1735 }
1736
1737 /* ABORT_REQ sent as a ULP command looks like this */
1738 #define LEN__ABORT_REQ_ULP (sizeof(struct ulp_txpkt) + \
1739         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_req_core))
1740
1741 static void *
1742 mk_abort_req_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1743 {
1744         struct ulptx_idata *ulpsc;
1745         struct cpl_abort_req_core *req;
1746
1747         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1748         ulpmc->len = htobe32(howmany(LEN__ABORT_REQ_ULP, 16));
1749
1750         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1751         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1752         ulpsc->len = htobe32(sizeof(*req));
1753
1754         req = (struct cpl_abort_req_core *)(ulpsc + 1);
1755         OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
1756         req->rsvd0 = htonl(0);
1757         req->rsvd1 = 0;
1758         req->cmd = CPL_ABORT_NO_RST;
1759
1760         ulpsc = (struct ulptx_idata *)(req + 1);
1761         if (LEN__ABORT_REQ_ULP % 16) {
1762                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1763                 ulpsc->len = htobe32(0);
1764                 return (ulpsc + 1);
1765         }
1766         return (ulpsc);
1767 }
1768
1769 /* ABORT_RPL sent as a ULP command looks like this */
1770 #define LEN__ABORT_RPL_ULP (sizeof(struct ulp_txpkt) + \
1771         sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_rpl_core))
1772
1773 static void *
1774 mk_abort_rpl_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1775 {
1776         struct ulptx_idata *ulpsc;
1777         struct cpl_abort_rpl_core *rpl;
1778
1779         ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1780         ulpmc->len = htobe32(howmany(LEN__ABORT_RPL_ULP, 16));
1781
1782         ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1783         ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1784         ulpsc->len = htobe32(sizeof(*rpl));
1785
1786         rpl = (struct cpl_abort_rpl_core *)(ulpsc + 1);
1787         OPCODE_TID(rpl) = htobe32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
1788         rpl->rsvd0 = htonl(0);
1789         rpl->rsvd1 = 0;
1790         rpl->cmd = CPL_ABORT_NO_RST;
1791
1792         ulpsc = (struct ulptx_idata *)(rpl + 1);
1793         if (LEN__ABORT_RPL_ULP % 16) {
1794                 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1795                 ulpsc->len = htobe32(0);
1796                 return (ulpsc + 1);
1797         }
1798         return (ulpsc);
1799 }
1800
1801 static inline int
1802 del_hashfilter_wrlen(void)
1803 {
1804
1805         return (sizeof(struct work_request_hdr) +
1806             roundup2(LEN__SET_TCB_FIELD_ULP, 16) +
1807             roundup2(LEN__ABORT_REQ_ULP, 16) +
1808             roundup2(LEN__ABORT_RPL_ULP, 16));
1809 }
1810
1811 static void
1812 mk_del_hashfilter_wr(int tid, struct work_request_hdr *wrh, int wrlen, int qid)
1813 {
1814         struct ulp_txpkt *ulpmc;
1815
1816         INIT_ULPTX_WRH(wrh, wrlen, 0, 0);
1817         ulpmc = (struct ulp_txpkt *)(wrh + 1);
1818         ulpmc = mk_set_tcb_field_ulp(ulpmc, W_TCB_RSS_INFO,
1819             V_TCB_RSS_INFO(M_TCB_RSS_INFO), V_TCB_RSS_INFO(qid), tid, 0);
1820         ulpmc = mk_abort_req_ulp(ulpmc, tid);
1821         ulpmc = mk_abort_rpl_ulp(ulpmc, tid);
1822 }
1823
1824 static int
1825 del_hashfilter(struct adapter *sc, struct t4_filter *t)
1826 {
1827         struct tid_info *ti = &sc->tids;
1828         void *wr;
1829         struct filter_entry *f;
1830         struct wrq_cookie cookie;
1831         int rc;
1832         const int wrlen = del_hashfilter_wrlen();
1833         const int inv_tid = ti->ntids + ti->tid_base;
1834
1835         MPASS(sc->tids.hftid_hash_4t != NULL);
1836         MPASS(sc->tids.ntids > 0);
1837
1838         if (t->idx < sc->tids.tid_base || t->idx >= inv_tid)
1839                 return (EINVAL);
1840
1841         mtx_lock(&ti->hftid_lock);
1842         f = lookup_hftid(sc, t->idx);
1843         if (f == NULL || f->valid == 0) {
1844                 rc = EINVAL;
1845                 goto done;
1846         }
1847         MPASS(f->tid == t->idx);
1848         if (f->locked) {
1849                 rc = EPERM;
1850                 goto done;
1851         }
1852         if (f->pending) {
1853                 rc = EBUSY;
1854                 goto done;
1855         }
1856         wr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(wrlen, 16), &cookie);
1857         if (wr == NULL) {
1858                 rc = ENOMEM;
1859                 goto done;
1860         }
1861
1862         mk_del_hashfilter_wr(t->idx, wr, wrlen, sc->sge.fwq.abs_id);
1863         f->locked = 1;
1864         f->pending = 1;
1865         commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie);
1866         t->fs = f->fs;  /* extra info for the caller */
1867
1868         for (;;) {
1869                 MPASS(f->locked);
1870                 if (f->pending == 0) {
1871                         if (f->valid) {
1872                                 f->locked = 0;
1873                                 rc = EIO;
1874                         } else {
1875                                 rc = 0;
1876                                 free(f, M_CXGBE);
1877                         }
1878                         break;
1879                 }
1880                 if (cv_wait_sig(&ti->hftid_cv, &ti->hftid_lock) != 0) {
1881                         f->locked = 0;
1882                         rc = EINPROGRESS;
1883                         break;
1884                 }
1885         }
1886 done:
1887         mtx_unlock(&ti->hftid_lock);
1888         return (rc);
1889 }
1890
1891 #define WORD_MASK       0xffffffff
1892 static void
1893 set_nat_params(struct adapter *sc, struct filter_entry *f, const bool dip,
1894     const bool sip, const bool dp, const bool sp)
1895 {
1896
1897         if (dip) {
1898                 if (f->fs.type) {
1899                         set_tcb_field(sc, f->tid, W_TCB_SND_UNA_RAW, WORD_MASK,
1900                             f->fs.nat_dip[15] | f->fs.nat_dip[14] << 8 |
1901                             f->fs.nat_dip[13] << 16 | f->fs.nat_dip[12] << 24, 1);
1902
1903                         set_tcb_field(sc, f->tid,
1904                             W_TCB_SND_UNA_RAW + 1, WORD_MASK,
1905                             f->fs.nat_dip[11] | f->fs.nat_dip[10] << 8 |
1906                             f->fs.nat_dip[9] << 16 | f->fs.nat_dip[8] << 24, 1);
1907
1908                         set_tcb_field(sc, f->tid,
1909                             W_TCB_SND_UNA_RAW + 2, WORD_MASK,
1910                             f->fs.nat_dip[7] | f->fs.nat_dip[6] << 8 |
1911                             f->fs.nat_dip[5] << 16 | f->fs.nat_dip[4] << 24, 1);
1912
1913                         set_tcb_field(sc, f->tid,
1914                             W_TCB_SND_UNA_RAW + 3, WORD_MASK,
1915                             f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
1916                             f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
1917                 } else {
1918                         set_tcb_field(sc, f->tid,
1919                             W_TCB_RX_FRAG3_LEN_RAW, WORD_MASK,
1920                             f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
1921                             f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
1922                 }
1923         }
1924
1925         if (sip) {
1926                 if (f->fs.type) {
1927                         set_tcb_field(sc, f->tid,
1928                             W_TCB_RX_FRAG2_PTR_RAW, WORD_MASK,
1929                             f->fs.nat_sip[15] | f->fs.nat_sip[14] << 8 |
1930                             f->fs.nat_sip[13] << 16 | f->fs.nat_sip[12] << 24, 1);
1931
1932                         set_tcb_field(sc, f->tid,
1933                             W_TCB_RX_FRAG2_PTR_RAW + 1, WORD_MASK,
1934                             f->fs.nat_sip[11] | f->fs.nat_sip[10] << 8 |
1935                             f->fs.nat_sip[9] << 16 | f->fs.nat_sip[8] << 24, 1);
1936
1937                         set_tcb_field(sc, f->tid,
1938                             W_TCB_RX_FRAG2_PTR_RAW + 2, WORD_MASK,
1939                             f->fs.nat_sip[7] | f->fs.nat_sip[6] << 8 |
1940                             f->fs.nat_sip[5] << 16 | f->fs.nat_sip[4] << 24, 1);
1941
1942                         set_tcb_field(sc, f->tid,
1943                             W_TCB_RX_FRAG2_PTR_RAW + 3, WORD_MASK,
1944                             f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
1945                             f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
1946
1947                 } else {
1948                         set_tcb_field(sc, f->tid,
1949                             W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW, WORD_MASK,
1950                             f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
1951                             f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
1952                 }
1953         }
1954
1955         set_tcb_field(sc, f->tid, W_TCB_PDU_HDR_LEN, WORD_MASK,
1956             (dp ? f->fs.nat_dport : 0) | (sp ? f->fs.nat_sport << 16 : 0), 1);
1957 }
1958
1959 /*
1960  * Returns EINPROGRESS to indicate that at least one TCB update was sent and the
1961  * last of the series of updates requested a reply.  The reply informs the
1962  * driver that the filter is fully setup.
1963  */
1964 static int
1965 configure_hashfilter_tcb(struct adapter *sc, struct filter_entry *f)
1966 {
1967         int updated = 0;
1968
1969         MPASS(f->tid < sc->tids.ntids);
1970         MPASS(f->fs.hash);
1971         MPASS(f->pending);
1972         MPASS(f->valid == 0);
1973
1974         if (f->fs.newdmac) {
1975                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECE, 1, 1);
1976                 updated++;
1977         }
1978
1979         if (f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) {
1980                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_RFR, 1, 1);
1981                 updated++;
1982         }
1983
1984         if (f->fs.newsmac) {
1985                 MPASS(f->smt != NULL);
1986                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
1987                 set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
1988                     V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), V_TCB_SMAC_SEL(f->smt->idx),
1989                     1);
1990                 updated++;
1991         }
1992
1993         switch(f->fs.nat_mode) {
1994         case NAT_MODE_NONE:
1995                 break;
1996         case NAT_MODE_DIP:
1997                 set_nat_params(sc, f, true, false, false, false);
1998                 updated++;
1999                 break;
2000         case NAT_MODE_DIP_DP:
2001                 set_nat_params(sc, f, true, false, true, false);
2002                 updated++;
2003                 break;
2004         case NAT_MODE_DIP_DP_SIP:
2005                 set_nat_params(sc, f, true, true, true, false);
2006                 updated++;
2007                 break;
2008         case NAT_MODE_DIP_DP_SP:
2009                 set_nat_params(sc, f, true, false, true, true);
2010                 updated++;
2011                 break;
2012         case NAT_MODE_SIP_SP:
2013                 set_nat_params(sc, f, false, true, false, true);
2014                 updated++;
2015                 break;
2016         case NAT_MODE_DIP_SIP_SP:
2017                 set_nat_params(sc, f, true, true, false, true);
2018                 updated++;
2019                 break;
2020         case NAT_MODE_ALL:
2021                 set_nat_params(sc, f, true, true, true, true);
2022                 updated++;
2023                 break;
2024         default:
2025                 MPASS(0);       /* should have been validated earlier */
2026                 break;
2027
2028         }
2029
2030         if (f->fs.nat_seq_chk) {
2031                 set_tcb_field(sc, f->tid, W_TCB_RCV_NXT,
2032                     V_TCB_RCV_NXT(M_TCB_RCV_NXT),
2033                     V_TCB_RCV_NXT(f->fs.nat_seq_chk), 1);
2034                 updated++;
2035         }
2036
2037         if (is_t5(sc) && f->fs.action == FILTER_DROP) {
2038                 /*
2039                  * Migrating = 1, Non-offload = 0 to get a T5 hashfilter to drop.
2040                  */
2041                 set_tcb_field(sc, f->tid, W_TCB_T_FLAGS, V_TF_NON_OFFLOAD(1) |
2042                     V_TF_MIGRATING(1), V_TF_MIGRATING(1), 1);
2043                 updated++;
2044         }
2045
2046         /*
2047          * Enable switching after all secondary resources (L2T entry, SMT entry,
2048          * etc.) are setup so that any switched packet will use correct
2049          * values.
2050          */
2051         if (f->fs.action == FILTER_SWITCH) {
2052                 set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECN, 1, 1);
2053                 updated++;
2054         }
2055
2056         if (f->fs.hitcnts || updated > 0) {
2057                 set_tcb_field(sc, f->tid, W_TCB_TIMESTAMP,
2058                     V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) |
2059                     V_TCB_T_RTT_TS_RECENT_AGE(M_TCB_T_RTT_TS_RECENT_AGE),
2060                     V_TCB_TIMESTAMP(0ULL) | V_TCB_T_RTT_TS_RECENT_AGE(0ULL), 0);
2061                 return (EINPROGRESS);
2062         }
2063
2064         return (0);
2065 }