sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "efsys.h"
  35 #include "efx.h"
  36 #include "efx_types.h"
  37 #include "efx_regs.h"
  38 #include "efx_impl.h"
  39
  40
  41 #if EFSYS_OPT_FILTER
  42
  43 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
  44
  45 static  __checkReturn   int
  46 falconsiena_filter_init(
  47         __in            efx_nic_t *enp);
  48
  49 static                  void
  50 falconsiena_filter_fini(
  51         __in            efx_nic_t *enp);
  52
  53 static  __checkReturn   int
  54 falconsiena_filter_restore(
  55         __in            efx_nic_t *enp);
  56
  57 static  __checkReturn   int
  58 falconsiena_filter_add(
  59         __in            efx_nic_t *enp,
  60         __inout         efx_filter_spec_t *spec,
  61         __in            boolean_t may_replace);
  62
  63 static  __checkReturn   int
  64 falconsiena_filter_delete(
  65         __in            efx_nic_t *enp,
  66         __inout         efx_filter_spec_t *spec);
  67
  68 static  __checkReturn   int
  69 falconsiena_filter_supported_filters(
  70         __in            efx_nic_t *enp,
  71         __out           uint32_t *list,
  72         __out           size_t *length);
  73
  74 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
  75
  76 #if EFSYS_OPT_FALCON
  77 static efx_filter_ops_t __efx_filter_falcon_ops = {
  78         falconsiena_filter_init,                /* efo_init */
  79         falconsiena_filter_fini,                /* efo_fini */
  80         falconsiena_filter_restore,             /* efo_restore */
  81         falconsiena_filter_add,                 /* efo_add */
  82         falconsiena_filter_delete,              /* efo_delete */
  83         falconsiena_filter_supported_filters,   /* efo_supported_filters */
  84         NULL,                                   /* efo_reconfigure */
  85 };
  86 #endif /* EFSYS_OPT_FALCON */
  87
  88 #if EFSYS_OPT_SIENA
  89 static efx_filter_ops_t __efx_filter_siena_ops = {
  90         falconsiena_filter_init,                /* efo_init */
  91         falconsiena_filter_fini,                /* efo_fini */
  92         falconsiena_filter_restore,             /* efo_restore */
  93         falconsiena_filter_add,                 /* efo_add */
  94         falconsiena_filter_delete,              /* efo_delete */
  95         falconsiena_filter_supported_filters,   /* efo_supported_filters */
  96         NULL,                                   /* efo_reconfigure */
  97 };
  98 #endif /* EFSYS_OPT_SIENA */
  99
 100 #if EFSYS_OPT_HUNTINGTON
 101 static efx_filter_ops_t __efx_filter_hunt_ops = {
 102         hunt_filter_init,               /* efo_init */
 103         hunt_filter_fini,               /* efo_fini */
 104         hunt_filter_restore,            /* efo_restore */
 105         hunt_filter_add,                /* efo_add */
 106         hunt_filter_delete,             /* efo_delete */
 107         hunt_filter_supported_filters,  /* efo_supported_filters */
 108         hunt_filter_reconfigure,        /* efo_reconfigure */
 109 };
 110 #endif /* EFSYS_OPT_HUNTINGTON */
 111
 112         __checkReturn   int
 113 efx_filter_insert(
 114         __in            efx_nic_t *enp,
 115         __inout         efx_filter_spec_t *spec)
 116 {
 117         efx_filter_ops_t *efop = enp->en_efop;
 118
 119         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 120         EFSYS_ASSERT3P(spec, !=, NULL);
 121         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 122
 123         return (efop->efo_add(enp, spec, B_FALSE));
 124 }
 125
 126         __checkReturn   int
 127 efx_filter_remove(
 128         __in            efx_nic_t *enp,
 129         __inout         efx_filter_spec_t *spec)
 130 {
 131         efx_filter_ops_t *efop = enp->en_efop;
 132
 133         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 134         EFSYS_ASSERT3P(spec, !=, NULL);
 135         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 136
 137 #if EFSYS_OPT_RX_SCALE
 138         spec->efs_rss_context = enp->en_rss_context;
 139 #endif
 140
 141         return (efop->efo_delete(enp, spec));
 142 }
 143
 144         __checkReturn   int
 145 efx_filter_restore(
 146         __in            efx_nic_t *enp)
 147 {
 148         int rc;
 149
 150         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 151
 152         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 153                 goto fail1;
 154
 155         return (0);
 156
 157 fail1:
 158         EFSYS_PROBE1(fail1, int, rc);
 159
 160         return (rc);
 161 }
 162
 163         __checkReturn   int
 164 efx_filter_init(
 165         __in            efx_nic_t *enp)
 166 {
 167         efx_filter_ops_t *efop;
 168         int rc;
 169
 170         /* Check that efx_filter_spec_t is 64 bytes. */
 171         EFX_STATIC_ASSERT(sizeof (efx_filter_spec_t) == 64);
 172
 173         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 174         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 175         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 176
 177         switch (enp->en_family) {
 178 #if EFSYS_OPT_FALCON
 179         case EFX_FAMILY_FALCON:
 180                 efop = (efx_filter_ops_t *)&__efx_filter_falcon_ops;
 181                 break;
 182 #endif /* EFSYS_OPT_FALCON */
 183
 184 #if EFSYS_OPT_SIENA
 185         case EFX_FAMILY_SIENA:
 186                 efop = (efx_filter_ops_t *)&__efx_filter_siena_ops;
 187                 break;
 188 #endif /* EFSYS_OPT_SIENA */
 189
 190 #if EFSYS_OPT_HUNTINGTON
 191         case EFX_FAMILY_HUNTINGTON:
 192                 efop = (efx_filter_ops_t *)&__efx_filter_hunt_ops;
 193                 break;
 194 #endif /* EFSYS_OPT_HUNTINGTON */
 195
 196         default:
 197                 EFSYS_ASSERT(0);
 198                 rc = ENOTSUP;
 199                 goto fail1;
 200         }
 201
 202         if ((rc = efop->efo_init(enp)) != 0)
 203                 goto fail2;
 204
 205         enp->en_efop = efop;
 206         enp->en_mod_flags |= EFX_MOD_FILTER;
 207         return (0);
 208
 209 fail2:
 210         EFSYS_PROBE(fail2);
 211 fail1:
 212         EFSYS_PROBE1(fail1, int, rc);
 213
 214         enp->en_efop = NULL;
 215         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 216         return (rc);
 217 }
 218
 219                         void
 220 efx_filter_fini(
 221         __in            efx_nic_t *enp)
 222 {
 223         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 224         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 225         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 226
 227         enp->en_efop->efo_fini(enp);
 228
 229         enp->en_efop = NULL;
 230         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 231 }
 232
 233         __checkReturn   int
 234 efx_filter_supported_filters(
 235         __in            efx_nic_t *enp,
 236         __out           uint32_t *list,
 237         __out           size_t *length)
 238 {
 239         int rc;
 240
 241         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 242         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 243         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 244         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 245
 246         if ((rc = enp->en_efop->efo_supported_filters(enp, list, length)) != 0)
 247                 goto fail1;
 248
 249         return (0);
 250
 251 fail1:
 252         EFSYS_PROBE1(fail1, int, rc);
 253
 254         return (rc);
 255 }
 256
 257         __checkReturn   int
 258 efx_filter_reconfigure(
 259         __in                            efx_nic_t *enp,
 260         __in_ecount(6)                  uint8_t const *mac_addr,
 261         __in                            boolean_t all_unicst,
 262         __in                            boolean_t mulcst,
 263         __in                            boolean_t all_mulcst,
 264         __in                            boolean_t brdcst,
 265         __in_ecount(6*count)            uint8_t const *addrs,
 266         __in                            int count)
 267 {
 268         int rc;
 269
 270         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 271         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 272         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 273
 274         if (enp->en_efop->efo_reconfigure != NULL) {
 275                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 276                                                         all_unicst, mulcst,
 277                                                         all_mulcst, brdcst,
 278                                                         addrs, count)) != 0)
 279                         goto fail1;
 280         }
 281
 282         return (0);
 283
 284 fail1:
 285         EFSYS_PROBE1(fail1, int, rc);
 286
 287         return (rc);
 288 }
 289
 290                 void
 291 efx_filter_spec_init_rx(
 292         __inout         efx_filter_spec_t *spec,
 293         __in            efx_filter_priority_t priority,
 294         __in            efx_filter_flag_t flags,
 295         __in            efx_rxq_t *erp)
 296 {
 297         EFSYS_ASSERT3P(spec, !=, NULL);
 298         EFSYS_ASSERT3P(erp, !=, NULL);
 299         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 300                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 301
 302         memset(spec, 0, sizeof (*spec));
 303         spec->efs_priority = priority;
 304         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 305         spec->efs_rss_context = EFX_FILTER_SPEC_RSS_CONTEXT_DEFAULT;
 306         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 307 }
 308
 309                 void
 310 efx_filter_spec_init_tx(
 311         __inout         efx_filter_spec_t *spec,
 312         __in            efx_txq_t *etp)
 313 {
 314         EFSYS_ASSERT3P(spec, !=, NULL);
 315         EFSYS_ASSERT3P(etp, !=, NULL);
 316
 317         memset(spec, 0, sizeof (*spec));
 318         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 319         spec->efs_flags = EFX_FILTER_FLAG_TX;
 320         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 321 }
 322
 323
 324 /*
 325  *  Specify IPv4 host, transport protocol and port in a filter specification
 326  */
 327 __checkReturn           int
 328 efx_filter_spec_set_ipv4_local(
 329         __inout         efx_filter_spec_t *spec,
 330         __in            uint8_t proto,
 331         __in            uint32_t host,
 332         __in            uint16_t port)
 333 {
 334         EFSYS_ASSERT3P(spec, !=, NULL);
 335
 336         spec->efs_match_flags |=
 337                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 338                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 339         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 340         spec->efs_ip_proto = proto;
 341         spec->efs_loc_host.eo_u32[0] = host;
 342         spec->efs_loc_port = port;
 343         return (0);
 344 }
 345
 346 /*
 347  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 348  */
 349 __checkReturn           int
 350 efx_filter_spec_set_ipv4_full(
 351         __inout         efx_filter_spec_t *spec,
 352         __in            uint8_t proto,
 353         __in            uint32_t lhost,
 354         __in            uint16_t lport,
 355         __in            uint32_t rhost,
 356         __in            uint16_t rport)
 357 {
 358         EFSYS_ASSERT3P(spec, !=, NULL);
 359
 360         spec->efs_match_flags |=
 361                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 362                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 363                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 364         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 365         spec->efs_ip_proto = proto;
 366         spec->efs_loc_host.eo_u32[0] = lhost;
 367         spec->efs_loc_port = lport;
 368         spec->efs_rem_host.eo_u32[0] = rhost;
 369         spec->efs_rem_port = rport;
 370         return (0);
 371 }
 372
 373 /*
 374  * Specify local Ethernet address and/or VID in filter specification
 375  */
 376 __checkReturn           int
 377 efx_filter_spec_set_eth_local(
 378         __inout         efx_filter_spec_t *spec,
 379         __in            uint16_t vid,
 380         __in            const uint8_t *addr)
 381 {
 382         EFSYS_ASSERT3P(spec, !=, NULL);
 383         EFSYS_ASSERT3P(addr, !=, NULL);
 384
 385         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 386                 return (EINVAL);
 387
 388         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 389                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 390                 spec->efs_outer_vid = vid;
 391         }
 392         if (addr != NULL) {
 393                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 394                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 395         }
 396         return (0);
 397 }
 398
 399 /*
 400  * Specify matching otherwise-unmatched unicast in a filter specification
 401  */
 402 __checkReturn           int
 403 efx_filter_spec_set_uc_def(
 404         __inout         efx_filter_spec_t *spec)
 405 {
 406         EFSYS_ASSERT3P(spec, !=, NULL);
 407
 408         spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC_IG;
 409         return (0);
 410 }
 411
 412 /*
 413  * Specify matching otherwise-unmatched multicast in a filter specification
 414  */
 415 __checkReturn           int
 416 efx_filter_spec_set_mc_def(
 417         __inout         efx_filter_spec_t *spec)
 418 {
 419         EFSYS_ASSERT3P(spec, !=, NULL);
 420
 421         spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC_IG;
 422         spec->efs_loc_mac[0] = 1;
 423         return (0);
 424 }
 425
 426
 427
 428 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
 429
 430 /*
 431  * "Fudge factors" - difference between programmed value and actual depth.
 432  * Due to pipelined implementation we need to program H/W with a value that
 433  * is larger than the hop limit we want.
 434  */
 435 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 436 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 437
 438 /*
 439  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 440  * We also need to avoid infinite loops in efx_filter_search() when the
 441  * table is full.
 442  */
 443 #define FILTER_CTL_SRCH_MAX 200
 444
 445 static  __checkReturn   int
 446 falconsiena_filter_spec_from_gen_spec(
 447         __out           falconsiena_filter_spec_t *fs_spec,
 448         __in            efx_filter_spec_t *gen_spec)
 449 {
 450         int rc;
 451         boolean_t is_full = B_FALSE;
 452
 453         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 454                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 455         else
 456                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 457
 458         /* Falconsiena only has one RSS context */
 459         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 460             gen_spec->efs_rss_context != 0) {
 461                 rc = EINVAL;
 462                 goto fail1;
 463         }
 464
 465         fs_spec->fsfs_flags = gen_spec->efs_flags;
 466         fs_spec->fsfs_dmaq_id = gen_spec->efs_dmaq_id;
 467
 468         switch (gen_spec->efs_match_flags) {
 469         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 470             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 471             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 472                 is_full = B_TRUE;
 473                 /* Fall through */
 474         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 475             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 476                 uint32_t rhost, host1, host2;
 477                 uint16_t rport, port1, port2;
 478
 479                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 480                         rc = ENOTSUP;
 481                         goto fail2;
 482                 }
 483                 if (gen_spec->efs_loc_port == 0 ||
 484                     (is_full && gen_spec->efs_rem_port == 0)) {
 485                         rc = EINVAL;
 486                         goto fail3;
 487                 }
 488                 switch (gen_spec->efs_ip_proto) {
 489                 case EFX_IPPROTO_TCP:
 490                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 491                                 fs_spec->fsfs_type = (is_full ?
 492                                     EFX_FS_FILTER_TX_TCP_FULL :
 493                                     EFX_FS_FILTER_TX_TCP_WILD);
 494                         } else {
 495                                 fs_spec->fsfs_type = (is_full ?
 496                                     EFX_FS_FILTER_RX_TCP_FULL :
 497                                     EFX_FS_FILTER_RX_TCP_WILD);
 498                         }
 499                         break;
 500                 case EFX_IPPROTO_UDP:
 501                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 502                                 fs_spec->fsfs_type = (is_full ?
 503                                     EFX_FS_FILTER_TX_UDP_FULL :
 504                                     EFX_FS_FILTER_TX_UDP_WILD);
 505                         } else {
 506                                 fs_spec->fsfs_type = (is_full ?
 507                                     EFX_FS_FILTER_RX_UDP_FULL :
 508                                     EFX_FS_FILTER_RX_UDP_WILD);
 509                         }
 510                         break;
 511                 default:
 512                         rc = ENOTSUP;
 513                         goto fail4;
 514                 }
 515                 /*
 516                  * The filter is constructed in terms of source and destination,
 517                  * with the odd wrinkle that the ports are swapped in a UDP
 518                  * wildcard filter. We need to convert from local and remote
 519                  * addresses (zero for a wildcard).
 520                  */
 521                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 522                 rport = is_full ? gen_spec->efs_rem_port : 0;
 523                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 524                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 525                         host2 = rhost;
 526                 } else {
 527                         host1 = rhost;
 528                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 529                 }
 530                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 531                         if (fs_spec->fsfs_type == EFX_FS_FILTER_TX_UDP_WILD) {
 532                                 port1 = rport;
 533                                 port2 = gen_spec->efs_loc_port;
 534                         } else {
 535                                 port1 = gen_spec->efs_loc_port;
 536                                 port2 = rport;
 537                         }
 538                 } else {
 539                         if (fs_spec->fsfs_type == EFX_FS_FILTER_RX_UDP_WILD) {
 540                                 port1 = gen_spec->efs_loc_port;
 541                                 port2 = rport;
 542                         } else {
 543                                 port1 = rport;
 544                                 port2 = gen_spec->efs_loc_port;
 545                         }
 546                 }
 547                 fs_spec->fsfs_dword[0] = (host1 << 16) | port1;
 548                 fs_spec->fsfs_dword[1] = (port2 << 16) | (host1 >> 16);
 549                 fs_spec->fsfs_dword[2] = host2;
 550                 break;
 551         }
 552
 553         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 554                 is_full = B_TRUE;
 555                 /* Fall through */
 556         case EFX_FILTER_MATCH_LOC_MAC:
 557                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 558                         fs_spec->fsfs_type = (is_full ?
 559                             EFX_FS_FILTER_TX_MAC_FULL :
 560                             EFX_FS_FILTER_TX_MAC_WILD);
 561                 } else {
 562                         fs_spec->fsfs_type = (is_full ?
 563                             EFX_FS_FILTER_RX_MAC_FULL :
 564                             EFX_FS_FILTER_RX_MAC_WILD);
 565                 }
 566                 fs_spec->fsfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 567                 fs_spec->fsfs_dword[1] =
 568                     gen_spec->efs_loc_mac[2] << 24 |
 569                     gen_spec->efs_loc_mac[3] << 16 |
 570                     gen_spec->efs_loc_mac[4] <<  8 |
 571                     gen_spec->efs_loc_mac[5];
 572                 fs_spec->fsfs_dword[2] =
 573                     gen_spec->efs_loc_mac[0] << 8 |
 574                     gen_spec->efs_loc_mac[1];
 575                 break;
 576
 577         default:
 578                 EFSYS_ASSERT(B_FALSE);
 579                 rc = ENOTSUP;
 580                 goto fail5;
 581         }
 582
 583         return (0);
 584
 585 fail5:
 586         EFSYS_PROBE(fail5);
 587 fail4:
 588         EFSYS_PROBE(fail4);
 589 fail3:
 590         EFSYS_PROBE(fail3);
 591 fail2:
 592         EFSYS_PROBE(fail2);
 593 fail1:
 594         EFSYS_PROBE1(fail1, int, rc);
 595
 596         return (rc);
 597 }
 598
 599 /*
 600  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 601  * key derived from the n-tuple.
 602  */
 603 static                  uint16_t
 604 falconsiena_filter_tbl_hash(
 605         __in            uint32_t key)
 606 {
 607         uint16_t tmp;
 608
 609         /* First 16 rounds */
 610         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 611         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 612         tmp = tmp ^ tmp >> 9;
 613
 614         /* Last 16 rounds */
 615         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 616         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 617         tmp = tmp ^ tmp >> 9;
 618
 619         return (tmp);
 620 }
 621
 622 /*
 623  * To allow for hash collisions, filter search continues at these
 624  * increments from the first possible entry selected by the hash.
 625  */
 626 static                  uint16_t
 627 falconsiena_filter_tbl_increment(
 628         __in            uint32_t key)
 629 {
 630         return ((uint16_t)(key * 2 - 1));
 631 }
 632
 633 static  __checkReturn   boolean_t
 634 falconsiena_filter_test_used(
 635         __in            falconsiena_filter_tbl_t *fsftp,
 636         __in            unsigned int index)
 637 {
 638         EFSYS_ASSERT3P(fsftp->fsft_bitmap, !=, NULL);
 639         return ((fsftp->fsft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 640 }
 641
 642 static                  void
 643 falconsiena_filter_set_used(
 644         __in            falconsiena_filter_tbl_t *fsftp,
 645         __in            unsigned int index)
 646 {
 647         EFSYS_ASSERT3P(fsftp->fsft_bitmap, !=, NULL);
 648         fsftp->fsft_bitmap[index / 32] |= (1 << (index % 32));
 649         ++fsftp->fsft_used;
 650 }
 651
 652 static                  void
 653 falconsiena_filter_clear_used(
 654         __in            falconsiena_filter_tbl_t *fsftp,
 655         __in            unsigned int index)
 656 {
 657         EFSYS_ASSERT3P(fsftp->fsft_bitmap, !=, NULL);
 658         fsftp->fsft_bitmap[index / 32] &= ~(1 << (index % 32));
 659
 660         --fsftp->fsft_used;
 661         EFSYS_ASSERT3U(fsftp->fsft_used, >=, 0);
 662 }
 663
 664
 665 static                  falconsiena_filter_tbl_id_t
 666 falconsiena_filter_tbl_id(
 667         __in            falconsiena_filter_type_t type)
 668 {
 669         falconsiena_filter_tbl_id_t tbl_id;
 670
 671         switch (type) {
 672         case EFX_FS_FILTER_RX_TCP_FULL:
 673         case EFX_FS_FILTER_RX_TCP_WILD:
 674         case EFX_FS_FILTER_RX_UDP_FULL:
 675         case EFX_FS_FILTER_RX_UDP_WILD:
 676                 tbl_id = EFX_FS_FILTER_TBL_RX_IP;
 677                 break;
 678
 679 #if EFSYS_OPT_SIENA
 680         case EFX_FS_FILTER_RX_MAC_FULL:
 681         case EFX_FS_FILTER_RX_MAC_WILD:
 682                 tbl_id = EFX_FS_FILTER_TBL_RX_MAC;
 683                 break;
 684
 685         case EFX_FS_FILTER_TX_TCP_FULL:
 686         case EFX_FS_FILTER_TX_TCP_WILD:
 687         case EFX_FS_FILTER_TX_UDP_FULL:
 688         case EFX_FS_FILTER_TX_UDP_WILD:
 689                 tbl_id = EFX_FS_FILTER_TBL_TX_IP;
 690                 break;
 691
 692         case EFX_FS_FILTER_TX_MAC_FULL:
 693         case EFX_FS_FILTER_TX_MAC_WILD:
 694                 tbl_id = EFX_FS_FILTER_TBL_TX_MAC;
 695                 break;
 696 #endif  /* EFSYS_OPT_SIENA */
 697
 698         default:
 699                 EFSYS_ASSERT(B_FALSE);
 700                 tbl_id = EFX_FS_FILTER_NTBLS;
 701                 break;
 702         }
 703         return (tbl_id);
 704 }
 705
 706 static                  void
 707 falconsiena_filter_reset_search_depth(
 708         __inout         falconsiena_filter_t *fsfp,
 709         __in            falconsiena_filter_tbl_id_t tbl_id)
 710 {
 711         switch (tbl_id) {
 712         case EFX_FS_FILTER_TBL_RX_IP:
 713                 fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_FULL] = 0;
 714                 fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_WILD] = 0;
 715                 fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_FULL] = 0;
 716                 fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_WILD] = 0;
 717                 break;
 718
 719 #if EFSYS_OPT_SIENA
 720         case EFX_FS_FILTER_TBL_RX_MAC:
 721                 fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_FULL] = 0;
 722                 fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_WILD] = 0;
 723                 break;
 724
 725         case EFX_FS_FILTER_TBL_TX_IP:
 726                 fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_FULL] = 0;
 727                 fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_WILD] = 0;
 728                 fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_FULL] = 0;
 729                 fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_WILD] = 0;
 730                 break;
 731
 732         case EFX_FS_FILTER_TBL_TX_MAC:
 733                 fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_FULL] = 0;
 734                 fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_WILD] = 0;
 735                 break;
 736 #endif  /* EFSYS_OPT_SIENA */
 737
 738         default:
 739                 EFSYS_ASSERT(B_FALSE);
 740                 break;
 741         }
 742 }
 743
 744 static                  void
 745 falconsiena_filter_push_rx_limits(
 746         __in            efx_nic_t *enp)
 747 {
 748         falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
 749         efx_oword_t oword;
 750
 751         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 752
 753         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 754             fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_FULL] +
 755             FILTER_CTL_SRCH_FUDGE_FULL);
 756         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 757             fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_WILD] +
 758             FILTER_CTL_SRCH_FUDGE_WILD);
 759         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 760             fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_FULL] +
 761             FILTER_CTL_SRCH_FUDGE_FULL);
 762         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 763             fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_WILD] +
 764             FILTER_CTL_SRCH_FUDGE_WILD);
 765
 766 #if EFSYS_OPT_SIENA
 767         if (fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_MAC].fsft_size) {
 768                 EFX_SET_OWORD_FIELD(oword,
 769                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 770                     fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_FULL] +
 771                     FILTER_CTL_SRCH_FUDGE_FULL);
 772                 EFX_SET_OWORD_FIELD(oword,
 773                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 774                     fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_WILD] +
 775                     FILTER_CTL_SRCH_FUDGE_WILD);
 776         }
 777 #endif /* EFSYS_OPT_SIENA */
 778
 779         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 780 }
 781
 782 static                  void
 783 falconsiena_filter_push_tx_limits(
 784         __in            efx_nic_t *enp)
 785 {
 786         falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
 787         efx_oword_t oword;
 788
 789         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 790
 791         if (fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_IP].fsft_size != 0) {
 792                 EFX_SET_OWORD_FIELD(oword,
 793                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 794                     fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_FULL] +
 795                     FILTER_CTL_SRCH_FUDGE_FULL);
 796                 EFX_SET_OWORD_FIELD(oword,
 797                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 798                     fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_WILD] +
 799                     FILTER_CTL_SRCH_FUDGE_WILD);
 800                 EFX_SET_OWORD_FIELD(oword,
 801                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 802                     fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_FULL] +
 803                     FILTER_CTL_SRCH_FUDGE_FULL);
 804                 EFX_SET_OWORD_FIELD(oword,
 805                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 806                     fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_WILD] +
 807                     FILTER_CTL_SRCH_FUDGE_WILD);
 808         }
 809
 810         if (fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_MAC].fsft_size != 0) {
 811                 EFX_SET_OWORD_FIELD(
 812                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 813                         fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_FULL] +
 814                         FILTER_CTL_SRCH_FUDGE_FULL);
 815                 EFX_SET_OWORD_FIELD(
 816                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 817                         fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_WILD] +
 818                         FILTER_CTL_SRCH_FUDGE_WILD);
 819         }
 820
 821         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 822 }
 823
 824 /* Build a filter entry and return its n-tuple key. */
 825 static  __checkReturn   uint32_t
 826 falconsiena_filter_build(
 827         __out           efx_oword_t *filter,
 828         __in            falconsiena_filter_spec_t *spec)
 829 {
 830         uint32_t dword3;
 831         uint32_t key;
 832         uint8_t  type  = spec->fsfs_type;
 833         uint32_t flags = spec->fsfs_flags;
 834
 835         switch (falconsiena_filter_tbl_id(type)) {
 836         case EFX_FS_FILTER_TBL_RX_IP: {
 837                 boolean_t is_udp = (type == EFX_FS_FILTER_RX_UDP_FULL ||
 838                     type == EFX_FS_FILTER_RX_UDP_WILD);
 839                 EFX_POPULATE_OWORD_7(*filter,
 840                     FRF_BZ_RSS_EN,
 841                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 842                     FRF_BZ_SCATTER_EN,
 843                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 844                     FRF_AZ_TCP_UDP, is_udp,
 845                     FRF_AZ_RXQ_ID, spec->fsfs_dmaq_id,
 846                     EFX_DWORD_2, spec->fsfs_dword[2],
 847                     EFX_DWORD_1, spec->fsfs_dword[1],
 848                     EFX_DWORD_0, spec->fsfs_dword[0]);
 849                 dword3 = is_udp;
 850                 break;
 851         }
 852
 853 #if EFSYS_OPT_SIENA
 854         case EFX_FS_FILTER_TBL_RX_MAC: {
 855                 boolean_t is_wild = (type == EFX_FS_FILTER_RX_MAC_WILD);
 856                 EFX_POPULATE_OWORD_7(*filter,
 857                     FRF_CZ_RMFT_RSS_EN,
 858                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 859                     FRF_CZ_RMFT_SCATTER_EN,
 860                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 861                     FRF_CZ_RMFT_RXQ_ID, spec->fsfs_dmaq_id,
 862                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 863                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->fsfs_dword[2],
 864                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->fsfs_dword[1],
 865                     FRF_CZ_RMFT_VLAN_ID, spec->fsfs_dword[0]);
 866                 dword3 = is_wild;
 867                 break;
 868         }
 869 #endif /* EFSYS_OPT_SIENA */
 870
 871         case EFX_FS_FILTER_TBL_TX_IP: {
 872                 boolean_t is_udp = (type == EFX_FS_FILTER_TX_UDP_FULL ||
 873                     type == EFX_FS_FILTER_TX_UDP_WILD);
 874                 EFX_POPULATE_OWORD_5(*filter,
 875                     FRF_CZ_TIFT_TCP_UDP, is_udp,
 876                     FRF_CZ_TIFT_TXQ_ID, spec->fsfs_dmaq_id,
 877                     EFX_DWORD_2, spec->fsfs_dword[2],
 878                     EFX_DWORD_1, spec->fsfs_dword[1],
 879                     EFX_DWORD_0, spec->fsfs_dword[0]);
 880                 dword3 = is_udp | spec->fsfs_dmaq_id << 1;
 881                 break;
 882         }
 883
 884 #if EFSYS_OPT_SIENA
 885         case EFX_FS_FILTER_TBL_TX_MAC: {
 886                 boolean_t is_wild = (type == EFX_FS_FILTER_TX_MAC_WILD);
 887                 EFX_POPULATE_OWORD_5(*filter,
 888                     FRF_CZ_TMFT_TXQ_ID, spec->fsfs_dmaq_id,
 889                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 890                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->fsfs_dword[2],
 891                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->fsfs_dword[1],
 892                     FRF_CZ_TMFT_VLAN_ID, spec->fsfs_dword[0]);
 893                 dword3 = is_wild | spec->fsfs_dmaq_id << 1;
 894                 break;
 895         }
 896 #endif /* EFSYS_OPT_SIENA */
 897
 898         default:
 899                 EFSYS_ASSERT(B_FALSE);
 900                 return (0);
 901         }
 902
 903         key =
 904             spec->fsfs_dword[0] ^
 905             spec->fsfs_dword[1] ^
 906             spec->fsfs_dword[2] ^
 907             dword3;
 908
 909         return (key);
 910 }
 911
 912 static  __checkReturn           int
 913 falconsiena_filter_push_entry(
 914         __inout                 efx_nic_t *enp,
 915         __in                    falconsiena_filter_type_t type,
 916         __in                    int index,
 917         __in                    efx_oword_t *eop)
 918 {
 919         int rc;
 920
 921         switch (type) {
 922         case EFX_FS_FILTER_RX_TCP_FULL:
 923         case EFX_FS_FILTER_RX_TCP_WILD:
 924         case EFX_FS_FILTER_RX_UDP_FULL:
 925         case EFX_FS_FILTER_RX_UDP_WILD:
 926                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
 927                     eop, B_TRUE);
 928                 break;
 929
 930 #if EFSYS_OPT_SIENA
 931         case EFX_FS_FILTER_RX_MAC_FULL:
 932         case EFX_FS_FILTER_RX_MAC_WILD:
 933                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
 934                     eop, B_TRUE);
 935                 break;
 936
 937         case EFX_FS_FILTER_TX_TCP_FULL:
 938         case EFX_FS_FILTER_TX_TCP_WILD:
 939         case EFX_FS_FILTER_TX_UDP_FULL:
 940         case EFX_FS_FILTER_TX_UDP_WILD:
 941                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
 942                     eop, B_TRUE);
 943                 break;
 944
 945         case EFX_FS_FILTER_TX_MAC_FULL:
 946         case EFX_FS_FILTER_TX_MAC_WILD:
 947                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
 948                     eop, B_TRUE);
 949                 break;
 950 #endif  /* EFSYS_OPT_SIENA */
 951
 952         default:
 953                 EFSYS_ASSERT(B_FALSE);
 954                 rc = ENOTSUP;
 955                 goto fail1;
 956         }
 957         return (0);
 958
 959 fail1:
 960         return (rc);
 961 }
 962
 963
 964 static  __checkReturn   boolean_t
 965 falconsiena_filter_equal(
 966         __in            const falconsiena_filter_spec_t *left,
 967         __in            const falconsiena_filter_spec_t *right)
 968 {
 969         falconsiena_filter_tbl_id_t tbl_id;
 970
 971         tbl_id = falconsiena_filter_tbl_id(left->fsfs_type);
 972
 973
 974         if (left->fsfs_type != right->fsfs_type)
 975                 return (B_FALSE);
 976
 977         if (memcmp(left->fsfs_dword, right->fsfs_dword,
 978                 sizeof (left->fsfs_dword)))
 979                 return (B_FALSE);
 980
 981         if ((tbl_id == EFX_FS_FILTER_TBL_TX_IP ||
 982                 tbl_id == EFX_FS_FILTER_TBL_TX_MAC) &&
 983             left->fsfs_dmaq_id != right->fsfs_dmaq_id)
 984                 return (B_FALSE);
 985
 986         return (B_TRUE);
 987 }
 988
 989 static  __checkReturn   int
 990 falconsiena_filter_search(
 991         __in            falconsiena_filter_tbl_t *fsftp,
 992         __in            falconsiena_filter_spec_t *spec,
 993         __in            uint32_t key,
 994         __in            boolean_t for_insert,
 995         __out           int *filter_index,
 996         __out           unsigned int *depth_required)
 997 {
 998         unsigned hash, incr, filter_idx, depth;
 999
1000         hash = falconsiena_filter_tbl_hash(key);
1001         incr = falconsiena_filter_tbl_increment(key);
1002
1003         filter_idx = hash & (fsftp->fsft_size - 1);
1004         depth = 1;
1005
1006         for (;;) {
1007                 /*
1008                  * Return success if entry is used and matches this spec
1009                  * or entry is unused and we are trying to insert.
1010                  */
1011                 if (falconsiena_filter_test_used(fsftp, filter_idx) ?
1012                     falconsiena_filter_equal(spec,
1013                     &fsftp->fsft_spec[filter_idx]) :
1014                     for_insert) {
1015                         *filter_index = filter_idx;
1016                         *depth_required = depth;
1017                         return (0);
1018                 }
1019
1020                 /* Return failure if we reached the maximum search depth */
1021                 if (depth == FILTER_CTL_SRCH_MAX)
1022                         return (for_insert ? EBUSY : ENOENT);
1023
1024                 filter_idx = (filter_idx + incr) & (fsftp->fsft_size - 1);
1025                 ++depth;
1026         }
1027 }
1028
1029 static                  void
1030 falconsiena_filter_clear_entry(
1031         __in            efx_nic_t *enp,
1032         __in            falconsiena_filter_tbl_t *fsftp,
1033         __in            int index)
1034 {
1035         efx_oword_t filter;
1036
1037         if (falconsiena_filter_test_used(fsftp, index)) {
1038                 falconsiena_filter_clear_used(fsftp, index);
1039
1040                 EFX_ZERO_OWORD(filter);
1041                 falconsiena_filter_push_entry(enp,
1042                     fsftp->fsft_spec[index].fsfs_type,
1043                     index, &filter);
1044
1045                 memset(&fsftp->fsft_spec[index],
1046                     0, sizeof (fsftp->fsft_spec[0]));
1047         }
1048 }
1049
1050                         void
1051 falconsiena_filter_tbl_clear(
1052         __in            efx_nic_t *enp,
1053         __in            falconsiena_filter_tbl_id_t tbl_id)
1054 {
1055         falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
1056         falconsiena_filter_tbl_t *fsftp = &fsfp->fsf_tbl[tbl_id];
1057         int index;
1058         int state;
1059
1060         EFSYS_LOCK(enp->en_eslp, state);
1061
1062         for (index = 0; index < fsftp->fsft_size; ++index) {
1063                 falconsiena_filter_clear_entry(enp, fsftp, index);
1064         }
1065
1066         if (fsftp->fsft_used == 0)
1067                 falconsiena_filter_reset_search_depth(fsfp, tbl_id);
1068
1069         EFSYS_UNLOCK(enp->en_eslp, state);
1070 }
1071
1072 static  __checkReturn   int
1073 falconsiena_filter_init(
1074         __in            efx_nic_t *enp)
1075 {
1076         falconsiena_filter_t *fsfp;
1077         falconsiena_filter_tbl_t *fsftp;
1078         int tbl_id;
1079         int rc;
1080
1081         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (falconsiena_filter_t), fsfp);
1082
1083         if (!fsfp) {
1084                 rc = ENOMEM;
1085                 goto fail1;
1086         }
1087
1088         enp->en_filter.ef_falconsiena_filter = fsfp;
1089
1090         switch (enp->en_family) {
1091 #if EFSYS_OPT_FALCON
1092         case EFX_FAMILY_FALCON:
1093                 fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_IP];
1094                 fsftp->fsft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1095                 break;
1096 #endif  /* EFSYS_OPT_FALCON */
1097
1098 #if EFSYS_OPT_SIENA
1099         case EFX_FAMILY_SIENA:
1100                 fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_IP];
1101                 fsftp->fsft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1102
1103                 fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_MAC];
1104                 fsftp->fsft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1105
1106                 fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_IP];
1107                 fsftp->fsft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1108
1109                 fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_MAC];
1110                 fsftp->fsft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1111                 break;
1112 #endif  /* EFSYS_OPT_SIENA */
1113
1114         default:
1115                 rc = ENOTSUP;
1116                 goto fail2;
1117         }
1118
1119         for (tbl_id = 0; tbl_id < EFX_FS_FILTER_NTBLS; tbl_id++) {
1120                 unsigned int bitmap_size;
1121
1122                 fsftp = &fsfp->fsf_tbl[tbl_id];
1123                 if (fsftp->fsft_size == 0)
1124                         continue;
1125
1126                 EFX_STATIC_ASSERT(sizeof (fsftp->fsft_bitmap[0]) ==
1127                     sizeof (uint32_t));
1128                 bitmap_size =
1129                     (fsftp->fsft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1130
1131                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, fsftp->fsft_bitmap);
1132                 if (!fsftp->fsft_bitmap) {
1133                         rc = ENOMEM;
1134                         goto fail3;
1135                 }
1136
1137                 EFSYS_KMEM_ALLOC(enp->en_esip,
1138                     fsftp->fsft_size * sizeof (*fsftp->fsft_spec),
1139                     fsftp->fsft_spec);
1140                 if (!fsftp->fsft_spec) {
1141                         rc = ENOMEM;
1142                         goto fail4;
1143                 }
1144                 memset(fsftp->fsft_spec, 0,
1145                     fsftp->fsft_size * sizeof (*fsftp->fsft_spec));
1146         }
1147
1148         return (0);
1149
1150 fail4:
1151         EFSYS_PROBE(fail4);
1152
1153 fail3:
1154         EFSYS_PROBE(fail3);
1155
1156 fail2:
1157         EFSYS_PROBE(fail2);
1158         falconsiena_filter_fini(enp);
1159
1160 fail1:
1161         EFSYS_PROBE1(fail1, int, rc);
1162         return (rc);
1163 }
1164
1165 static                  void
1166 falconsiena_filter_fini(
1167         __in            efx_nic_t *enp)
1168 {
1169         falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
1170         falconsiena_filter_tbl_id_t tbl_id;
1171
1172         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1173         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1174
1175         if (fsfp == NULL)
1176                 return;
1177
1178         for (tbl_id = 0; tbl_id < EFX_FS_FILTER_NTBLS; tbl_id++) {
1179                 falconsiena_filter_tbl_t *fsftp = &fsfp->fsf_tbl[tbl_id];
1180                 unsigned int bitmap_size;
1181
1182                 EFX_STATIC_ASSERT(sizeof (fsftp->fsft_bitmap[0]) ==
1183                     sizeof (uint32_t));
1184                 bitmap_size =
1185                     (fsftp->fsft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1186
1187                 if (fsftp->fsft_bitmap != NULL) {
1188                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1189                             fsftp->fsft_bitmap);
1190                         fsftp->fsft_bitmap = NULL;
1191                 }
1192
1193                 if (fsftp->fsft_spec != NULL) {
1194                         EFSYS_KMEM_FREE(enp->en_esip, fsftp->fsft_size *
1195                             sizeof (*fsftp->fsft_spec), fsftp->fsft_spec);
1196                         fsftp->fsft_spec = NULL;
1197                 }
1198         }
1199
1200         EFSYS_KMEM_FREE(enp->en_esip, sizeof (falconsiena_filter_t),
1201             enp->en_filter.ef_falconsiena_filter);
1202 }
1203
1204 /* Restore filter state after a reset */
1205 static  __checkReturn   int
1206 falconsiena_filter_restore(
1207         __in            efx_nic_t *enp)
1208 {
1209         falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
1210         falconsiena_filter_tbl_id_t tbl_id;
1211         falconsiena_filter_tbl_t *fsftp;
1212         falconsiena_filter_spec_t *spec;
1213         efx_oword_t filter;
1214         int filter_idx;
1215         int state;
1216         int rc;
1217
1218         EFSYS_LOCK(enp->en_eslp, state);
1219
1220         for (tbl_id = 0; tbl_id < EFX_FS_FILTER_NTBLS; tbl_id++) {
1221                 fsftp = &fsfp->fsf_tbl[tbl_id];
1222                 for (filter_idx = 0;
1223                         filter_idx < fsftp->fsft_size;
1224                         filter_idx++) {
1225                         if (!falconsiena_filter_test_used(fsftp, filter_idx))
1226                                 continue;
1227
1228                         spec = &fsftp->fsft_spec[filter_idx];
1229                         if ((rc = falconsiena_filter_build(&filter, spec)) != 0)
1230                                 goto fail1;
1231                         if ((rc = falconsiena_filter_push_entry(enp,
1232                                     spec->fsfs_type, filter_idx, &filter)) != 0)
1233                                 goto fail2;
1234                 }
1235         }
1236
1237         falconsiena_filter_push_rx_limits(enp);
1238         falconsiena_filter_push_tx_limits(enp);
1239
1240         EFSYS_UNLOCK(enp->en_eslp, state);
1241
1242         return (0);
1243
1244 fail2:
1245         EFSYS_PROBE(fail2);
1246
1247 fail1:
1248         EFSYS_PROBE1(fail1, int, rc);
1249
1250         EFSYS_UNLOCK(enp->en_eslp, state);
1251
1252         return (rc);
1253 }
1254
1255 static   __checkReturn  int
1256 falconsiena_filter_add(
1257         __in            efx_nic_t *enp,
1258         __inout         efx_filter_spec_t *spec,
1259         __in            boolean_t may_replace)
1260 {
1261         int rc;
1262         falconsiena_filter_spec_t fs_spec;
1263         falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
1264         falconsiena_filter_tbl_id_t tbl_id;
1265         falconsiena_filter_tbl_t *fsftp;
1266         falconsiena_filter_spec_t *saved_fs_spec;
1267         efx_oword_t filter;
1268         int filter_idx;
1269         unsigned int depth;
1270         int state;
1271         uint32_t key;
1272
1273
1274         EFSYS_ASSERT3P(spec, !=, NULL);
1275
1276         if ((rc = falconsiena_filter_spec_from_gen_spec(&fs_spec, spec)) != 0)
1277                 goto fail1;
1278
1279         tbl_id = falconsiena_filter_tbl_id(fs_spec.fsfs_type);
1280         fsftp = &fsfp->fsf_tbl[tbl_id];
1281
1282         if (fsftp->fsft_size == 0) {
1283                 rc = EINVAL;
1284                 goto fail2;
1285         }
1286
1287         key = falconsiena_filter_build(&filter, &fs_spec);
1288
1289         EFSYS_LOCK(enp->en_eslp, state);
1290
1291         rc = falconsiena_filter_search(fsftp, &fs_spec, key, B_TRUE,
1292             &filter_idx, &depth);
1293         if (rc != 0)
1294                 goto fail3;
1295
1296         EFSYS_ASSERT3U(filter_idx, <, fsftp->fsft_size);
1297         saved_fs_spec = &fsftp->fsft_spec[filter_idx];
1298
1299         if (falconsiena_filter_test_used(fsftp, filter_idx)) {
1300                 if (may_replace == B_FALSE) {
1301                         rc = EEXIST;
1302                         goto fail4;
1303                 }
1304         }
1305         falconsiena_filter_set_used(fsftp, filter_idx);
1306         *saved_fs_spec = fs_spec;
1307
1308         if (fsfp->fsf_depth[fs_spec.fsfs_type] < depth) {
1309                 fsfp->fsf_depth[fs_spec.fsfs_type] = depth;
1310                 if (tbl_id == EFX_FS_FILTER_TBL_TX_IP ||
1311                     tbl_id == EFX_FS_FILTER_TBL_TX_MAC)
1312                         falconsiena_filter_push_tx_limits(enp);
1313                 else
1314                         falconsiena_filter_push_rx_limits(enp);
1315         }
1316
1317         falconsiena_filter_push_entry(enp, fs_spec.fsfs_type,
1318             filter_idx, &filter);
1319
1320         EFSYS_UNLOCK(enp->en_eslp, state);
1321         return (0);
1322
1323 fail4:
1324         EFSYS_PROBE(fail4);
1325
1326 fail3:
1327         EFSYS_UNLOCK(enp->en_eslp, state);
1328         EFSYS_PROBE(fail3);
1329
1330 fail2:
1331         EFSYS_PROBE(fail2);
1332
1333 fail1:
1334         EFSYS_PROBE1(fail1, int, rc);
1335         return (rc);
1336 }
1337
1338 static   __checkReturn  int
1339 falconsiena_filter_delete(
1340         __in            efx_nic_t *enp,
1341         __inout         efx_filter_spec_t *spec)
1342 {
1343         int rc;
1344         falconsiena_filter_spec_t fs_spec;
1345         falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
1346         falconsiena_filter_tbl_id_t tbl_id;
1347         falconsiena_filter_tbl_t *fsftp;
1348         falconsiena_filter_spec_t *saved_spec;
1349         efx_oword_t filter;
1350         int filter_idx;
1351         unsigned int depth;
1352         int state;
1353         uint32_t key;
1354
1355         EFSYS_ASSERT3P(spec, !=, NULL);
1356
1357         if ((rc = falconsiena_filter_spec_from_gen_spec(&fs_spec, spec)) != 0)
1358                 goto fail1;
1359
1360         tbl_id = falconsiena_filter_tbl_id(fs_spec.fsfs_type);
1361         fsftp = &fsfp->fsf_tbl[tbl_id];
1362
1363         key = falconsiena_filter_build(&filter, &fs_spec);
1364
1365         EFSYS_LOCK(enp->en_eslp, state);
1366
1367         rc = falconsiena_filter_search(fsftp, &fs_spec, key, B_FALSE,
1368             &filter_idx, &depth);
1369         if (rc != 0)
1370                 goto fail2;
1371
1372         saved_spec = &fsftp->fsft_spec[filter_idx];
1373
1374         falconsiena_filter_clear_entry(enp, fsftp, filter_idx);
1375         if (fsftp->fsft_used == 0)
1376                 falconsiena_filter_reset_search_depth(fsfp, tbl_id);
1377
1378         EFSYS_UNLOCK(enp->en_eslp, state);
1379         return (0);
1380
1381 fail2:
1382         EFSYS_UNLOCK(enp->en_eslp, state);
1383         EFSYS_PROBE(fail2);
1384
1385 fail1:
1386         EFSYS_PROBE1(fail1, int, rc);
1387         return (rc);
1388 }
1389
1390 #define MAX_SUPPORTED 4
1391
1392 static  __checkReturn   int
1393 falconsiena_filter_supported_filters(
1394         __in            efx_nic_t *enp,
1395         __out           uint32_t *list,
1396         __out           size_t *length)
1397 {
1398         int index = 0;
1399         uint32_t rx_matches[MAX_SUPPORTED];
1400         int rc;
1401
1402         if (list == NULL) {
1403                 rc = EINVAL;
1404                 goto fail1;
1405         }
1406
1407         rx_matches[index++] =
1408             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1409             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1410             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1411
1412         rx_matches[index++] =
1413             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1414             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1415
1416         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1417                 rx_matches[index++] =
1418                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1419
1420                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1421         }
1422
1423         EFSYS_ASSERT3U(index, <=, MAX_SUPPORTED);
1424
1425         *length = index;
1426         memcpy(list, rx_matches, *length);
1427
1428         return (0);
1429
1430 fail1:
1431
1432         return (rc);
1433 }
1434
1435 #undef MAX_SUPPORTED
1436
1437 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1438
1439 #endif /* EFSYS_OPT_FILTER */