sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include "efx.h"
  37 #include "efx_impl.h"
  38
  39
  40 #if EFSYS_OPT_FILTER
  41
  42 #if EFSYS_OPT_SIENA
  43
  44 static  __checkReturn   efx_rc_t
  45 siena_filter_init(
  46         __in            efx_nic_t *enp);
  47
  48 static                  void
  49 siena_filter_fini(
  50         __in            efx_nic_t *enp);
  51
  52 static  __checkReturn   efx_rc_t
  53 siena_filter_restore(
  54         __in            efx_nic_t *enp);
  55
  56 static  __checkReturn   efx_rc_t
  57 siena_filter_add(
  58         __in            efx_nic_t *enp,
  59         __inout         efx_filter_spec_t *spec,
  60         __in            boolean_t may_replace);
  61
  62 static  __checkReturn   efx_rc_t
  63 siena_filter_delete(
  64         __in            efx_nic_t *enp,
  65         __inout         efx_filter_spec_t *spec);
  66
  67 static  __checkReturn   efx_rc_t
  68 siena_filter_supported_filters(
  69         __in                            efx_nic_t *enp,
  70         __out_ecount(buffer_length)     uint32_t *buffer,
  71         __in                            size_t buffer_length,
  72         __out                           size_t *list_lengthp);
  73
  74 #endif /* EFSYS_OPT_SIENA */
  75
  76 #if EFSYS_OPT_SIENA
  77 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  78         siena_filter_init,              /* efo_init */
  79         siena_filter_fini,              /* efo_fini */
  80         siena_filter_restore,           /* efo_restore */
  81         siena_filter_add,               /* efo_add */
  82         siena_filter_delete,            /* efo_delete */
  83         siena_filter_supported_filters, /* efo_supported_filters */
  84         NULL,                           /* efo_reconfigure */
  85 };
  86 #endif /* EFSYS_OPT_SIENA */
  87
  88 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  89 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  90         ef10_filter_init,               /* efo_init */
  91         ef10_filter_fini,               /* efo_fini */
  92         ef10_filter_restore,            /* efo_restore */
  93         ef10_filter_add,                /* efo_add */
  94         ef10_filter_delete,             /* efo_delete */
  95         ef10_filter_supported_filters,  /* efo_supported_filters */
  96         ef10_filter_reconfigure,        /* efo_reconfigure */
  97 };
  98 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
  99
 100         __checkReturn   efx_rc_t
 101 efx_filter_insert(
 102         __in            efx_nic_t *enp,
 103         __inout         efx_filter_spec_t *spec)
 104 {
 105         const efx_filter_ops_t *efop = enp->en_efop;
 106
 107         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 108         EFSYS_ASSERT3P(spec, !=, NULL);
 109         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 110
 111         return (efop->efo_add(enp, spec, B_FALSE));
 112 }
 113
 114         __checkReturn   efx_rc_t
 115 efx_filter_remove(
 116         __in            efx_nic_t *enp,
 117         __inout         efx_filter_spec_t *spec)
 118 {
 119         const efx_filter_ops_t *efop = enp->en_efop;
 120
 121         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 122         EFSYS_ASSERT3P(spec, !=, NULL);
 123         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 124
 125         return (efop->efo_delete(enp, spec));
 126 }
 127
 128         __checkReturn   efx_rc_t
 129 efx_filter_restore(
 130         __in            efx_nic_t *enp)
 131 {
 132         efx_rc_t rc;
 133
 134         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 135
 136         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 137                 goto fail1;
 138
 139         return (0);
 140
 141 fail1:
 142         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 143
 144         return (rc);
 145 }
 146
 147         __checkReturn   efx_rc_t
 148 efx_filter_init(
 149         __in            efx_nic_t *enp)
 150 {
 151         const efx_filter_ops_t *efop;
 152         efx_rc_t rc;
 153
 154         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 155         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 156         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 157
 158         switch (enp->en_family) {
 159 #if EFSYS_OPT_SIENA
 160         case EFX_FAMILY_SIENA:
 161                 efop = &__efx_filter_siena_ops;
 162                 break;
 163 #endif /* EFSYS_OPT_SIENA */
 164
 165 #if EFSYS_OPT_HUNTINGTON
 166         case EFX_FAMILY_HUNTINGTON:
 167                 efop = &__efx_filter_ef10_ops;
 168                 break;
 169 #endif /* EFSYS_OPT_HUNTINGTON */
 170
 171 #if EFSYS_OPT_MEDFORD
 172         case EFX_FAMILY_MEDFORD:
 173                 efop = &__efx_filter_ef10_ops;
 174                 break;
 175 #endif /* EFSYS_OPT_MEDFORD */
 176
 177         default:
 178                 EFSYS_ASSERT(0);
 179                 rc = ENOTSUP;
 180                 goto fail1;
 181         }
 182
 183         if ((rc = efop->efo_init(enp)) != 0)
 184                 goto fail2;
 185
 186         enp->en_efop = efop;
 187         enp->en_mod_flags |= EFX_MOD_FILTER;
 188         return (0);
 189
 190 fail2:
 191         EFSYS_PROBE(fail2);
 192 fail1:
 193         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 194
 195         enp->en_efop = NULL;
 196         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 197         return (rc);
 198 }
 199
 200                         void
 201 efx_filter_fini(
 202         __in            efx_nic_t *enp)
 203 {
 204         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 205         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 206         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 207
 208         enp->en_efop->efo_fini(enp);
 209
 210         enp->en_efop = NULL;
 211         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 212 }
 213
 214 /*
 215  * Query the possible combinations of match flags which can be filtered on.
 216  * These are returned as a list, of which each 32 bit element is a bitmask
 217  * formed of EFX_FILTER_MATCH flags.
 218  *
 219  * The combinations are ordered in priority from highest to lowest.
 220  *
 221  * If the provided buffer is too short to hold the list, the call with fail with
 222  * ENOSPC and *list_lengthp will be set to the buffer length required.
 223  */
 224         __checkReturn   efx_rc_t
 225 efx_filter_supported_filters(
 226         __in                            efx_nic_t *enp,
 227         __out_ecount(buffer_length)     uint32_t *buffer,
 228         __in                            size_t buffer_length,
 229         __out                           size_t *list_lengthp)
 230 {
 231         efx_rc_t rc;
 232
 233         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 234         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 235         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 236         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 237
 238         if (buffer == NULL) {
 239                 rc = EINVAL;
 240                 goto fail1;
 241         }
 242
 243         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 244                                                     list_lengthp);
 245         if (rc != 0)
 246                 goto fail2;
 247
 248         return (0);
 249
 250 fail2:
 251         EFSYS_PROBE(fail2);
 252 fail1:
 253         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 254
 255         return (rc);
 256 }
 257
 258         __checkReturn   efx_rc_t
 259 efx_filter_reconfigure(
 260         __in                            efx_nic_t *enp,
 261         __in_ecount(6)                  uint8_t const *mac_addr,
 262         __in                            boolean_t all_unicst,
 263         __in                            boolean_t mulcst,
 264         __in                            boolean_t all_mulcst,
 265         __in                            boolean_t brdcst,
 266         __in_ecount(6*count)            uint8_t const *addrs,
 267         __in                            uint32_t count)
 268 {
 269         efx_rc_t rc;
 270
 271         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 272         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 273         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 274
 275         if (enp->en_efop->efo_reconfigure != NULL) {
 276                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 277                                                         all_unicst, mulcst,
 278                                                         all_mulcst, brdcst,
 279                                                         addrs, count)) != 0)
 280                         goto fail1;
 281         }
 282
 283         return (0);
 284
 285 fail1:
 286         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 287
 288         return (rc);
 289 }
 290
 291                 void
 292 efx_filter_spec_init_rx(
 293         __out           efx_filter_spec_t *spec,
 294         __in            efx_filter_priority_t priority,
 295         __in            efx_filter_flags_t flags,
 296         __in            efx_rxq_t *erp)
 297 {
 298         EFSYS_ASSERT3P(spec, !=, NULL);
 299         EFSYS_ASSERT3P(erp, !=, NULL);
 300         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 301                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 302
 303         memset(spec, 0, sizeof (*spec));
 304         spec->efs_priority = priority;
 305         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 306         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 307         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 308 }
 309
 310                 void
 311 efx_filter_spec_init_tx(
 312         __out           efx_filter_spec_t *spec,
 313         __in            efx_txq_t *etp)
 314 {
 315         EFSYS_ASSERT3P(spec, !=, NULL);
 316         EFSYS_ASSERT3P(etp, !=, NULL);
 317
 318         memset(spec, 0, sizeof (*spec));
 319         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 320         spec->efs_flags = EFX_FILTER_FLAG_TX;
 321         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 322 }
 323
 324
 325 /*
 326  *  Specify IPv4 host, transport protocol and port in a filter specification
 327  */
 328 __checkReturn           efx_rc_t
 329 efx_filter_spec_set_ipv4_local(
 330         __inout         efx_filter_spec_t *spec,
 331         __in            uint8_t proto,
 332         __in            uint32_t host,
 333         __in            uint16_t port)
 334 {
 335         EFSYS_ASSERT3P(spec, !=, NULL);
 336
 337         spec->efs_match_flags |=
 338                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 339                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 340         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 341         spec->efs_ip_proto = proto;
 342         spec->efs_loc_host.eo_u32[0] = host;
 343         spec->efs_loc_port = port;
 344         return (0);
 345 }
 346
 347 /*
 348  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 349  */
 350 __checkReturn           efx_rc_t
 351 efx_filter_spec_set_ipv4_full(
 352         __inout         efx_filter_spec_t *spec,
 353         __in            uint8_t proto,
 354         __in            uint32_t lhost,
 355         __in            uint16_t lport,
 356         __in            uint32_t rhost,
 357         __in            uint16_t rport)
 358 {
 359         EFSYS_ASSERT3P(spec, !=, NULL);
 360
 361         spec->efs_match_flags |=
 362                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 363                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 364                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 365         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 366         spec->efs_ip_proto = proto;
 367         spec->efs_loc_host.eo_u32[0] = lhost;
 368         spec->efs_loc_port = lport;
 369         spec->efs_rem_host.eo_u32[0] = rhost;
 370         spec->efs_rem_port = rport;
 371         return (0);
 372 }
 373
 374 /*
 375  * Specify local Ethernet address and/or VID in filter specification
 376  */
 377 __checkReturn           efx_rc_t
 378 efx_filter_spec_set_eth_local(
 379         __inout         efx_filter_spec_t *spec,
 380         __in            uint16_t vid,
 381         __in            const uint8_t *addr)
 382 {
 383         EFSYS_ASSERT3P(spec, !=, NULL);
 384         EFSYS_ASSERT3P(addr, !=, NULL);
 385
 386         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 387                 return (EINVAL);
 388
 389         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 390                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 391                 spec->efs_outer_vid = vid;
 392         }
 393         if (addr != NULL) {
 394                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 395                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 396         }
 397         return (0);
 398 }
 399
 400                         void
 401 efx_filter_spec_set_ether_type(
 402         __inout         efx_filter_spec_t *spec,
 403         __in            uint16_t ether_type)
 404 {
 405         EFSYS_ASSERT3P(spec, !=, NULL);
 406
 407         spec->efs_ether_type = ether_type;
 408         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 409 }
 410
 411 /*
 412  * Specify matching otherwise-unmatched unicast in a filter specification
 413  */
 414 __checkReturn           efx_rc_t
 415 efx_filter_spec_set_uc_def(
 416         __inout         efx_filter_spec_t *spec)
 417 {
 418         EFSYS_ASSERT3P(spec, !=, NULL);
 419
 420         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 421         return (0);
 422 }
 423
 424 /*
 425  * Specify matching otherwise-unmatched multicast in a filter specification
 426  */
 427 __checkReturn           efx_rc_t
 428 efx_filter_spec_set_mc_def(
 429         __inout         efx_filter_spec_t *spec)
 430 {
 431         EFSYS_ASSERT3P(spec, !=, NULL);
 432
 433         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 434         return (0);
 435 }
 436
 437
 438 __checkReturn           efx_rc_t
 439 efx_filter_spec_set_encap_type(
 440         __inout         efx_filter_spec_t *spec,
 441         __in            efx_tunnel_protocol_t encap_type,
 442         __in            efx_filter_inner_frame_match_t inner_frame_match)
 443 {
 444         uint32_t match_flags = 0;
 445         uint8_t ip_proto;
 446         efx_rc_t rc;
 447
 448         EFSYS_ASSERT3P(spec, !=, NULL);
 449
 450         switch (encap_type) {
 451         case EFX_TUNNEL_PROTOCOL_VXLAN:
 452         case EFX_TUNNEL_PROTOCOL_GENEVE:
 453                 ip_proto = EFX_IPPROTO_UDP;
 454                 break;
 455         case EFX_TUNNEL_PROTOCOL_NVGRE:
 456                 ip_proto = EFX_IPPROTO_GRE;
 457                 break;
 458         default:
 459                 EFSYS_ASSERT(0);
 460                 rc = EINVAL;
 461                 goto fail1;
 462         }
 463
 464         switch (inner_frame_match) {
 465         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 466                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 467                 break;
 468         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 469                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 470                 break;
 471         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 472                 /* This is for when specific inner frames are to be matched. */
 473                 break;
 474         default:
 475                 EFSYS_ASSERT(0);
 476                 rc = EINVAL;
 477                 goto fail2;
 478         }
 479
 480         spec->efs_encap_type = encap_type;
 481         spec->efs_ip_proto = ip_proto;
 482         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 483
 484         return (0);
 485
 486 fail2:
 487         EFSYS_PROBE(fail2);
 488 fail1:
 489         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 490
 491         return (rc);
 492 }
 493
 494 #if EFSYS_OPT_RX_SCALE
 495         __checkReturn   efx_rc_t
 496 efx_filter_spec_set_rss_context(
 497         __inout         efx_filter_spec_t *spec,
 498         __in            uint32_t rss_context)
 499 {
 500         efx_rc_t rc;
 501
 502         EFSYS_ASSERT3P(spec, !=, NULL);
 503
 504         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 505         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 506                 rc = EINVAL;
 507                 goto fail1;
 508         }
 509
 510         spec->efs_rss_context = rss_context;
 511
 512         return (0);
 513
 514 fail1:
 515         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 516
 517         return (rc);
 518 }
 519 #endif
 520
 521 #if EFSYS_OPT_SIENA
 522
 523 /*
 524  * "Fudge factors" - difference between programmed value and actual depth.
 525  * Due to pipelined implementation we need to program H/W with a value that
 526  * is larger than the hop limit we want.
 527  */
 528 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 529 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 530
 531 /*
 532  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 533  * We also need to avoid infinite loops in efx_filter_search() when the
 534  * table is full.
 535  */
 536 #define FILTER_CTL_SRCH_MAX 200
 537
 538 static  __checkReturn   efx_rc_t
 539 siena_filter_spec_from_gen_spec(
 540         __out           siena_filter_spec_t *sf_spec,
 541         __in            efx_filter_spec_t *gen_spec)
 542 {
 543         efx_rc_t rc;
 544         boolean_t is_full = B_FALSE;
 545
 546         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 547                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 548         else
 549                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 550
 551         /* Siena only has one RSS context */
 552         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 553             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 554                 rc = EINVAL;
 555                 goto fail1;
 556         }
 557
 558         sf_spec->sfs_flags = gen_spec->efs_flags;
 559         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 560
 561         switch (gen_spec->efs_match_flags) {
 562         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 563             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 564             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 565                 is_full = B_TRUE;
 566                 /* Fall through */
 567         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 568             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 569                 uint32_t rhost, host1, host2;
 570                 uint16_t rport, port1, port2;
 571
 572                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 573                         rc = ENOTSUP;
 574                         goto fail2;
 575                 }
 576                 if (gen_spec->efs_loc_port == 0 ||
 577                     (is_full && gen_spec->efs_rem_port == 0)) {
 578                         rc = EINVAL;
 579                         goto fail3;
 580                 }
 581                 switch (gen_spec->efs_ip_proto) {
 582                 case EFX_IPPROTO_TCP:
 583                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 584                                 sf_spec->sfs_type = (is_full ?
 585                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 586                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 587                         } else {
 588                                 sf_spec->sfs_type = (is_full ?
 589                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 590                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 591                         }
 592                         break;
 593                 case EFX_IPPROTO_UDP:
 594                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 595                                 sf_spec->sfs_type = (is_full ?
 596                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 597                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 598                         } else {
 599                                 sf_spec->sfs_type = (is_full ?
 600                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 601                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 602                         }
 603                         break;
 604                 default:
 605                         rc = ENOTSUP;
 606                         goto fail4;
 607                 }
 608                 /*
 609                  * The filter is constructed in terms of source and destination,
 610                  * with the odd wrinkle that the ports are swapped in a UDP
 611                  * wildcard filter. We need to convert from local and remote
 612                  * addresses (zero for a wildcard).
 613                  */
 614                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 615                 rport = is_full ? gen_spec->efs_rem_port : 0;
 616                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 617                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 618                         host2 = rhost;
 619                 } else {
 620                         host1 = rhost;
 621                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 622                 }
 623                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 624                         if (sf_spec->sfs_type ==
 625                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 626                                 port1 = rport;
 627                                 port2 = gen_spec->efs_loc_port;
 628                         } else {
 629                                 port1 = gen_spec->efs_loc_port;
 630                                 port2 = rport;
 631                         }
 632                 } else {
 633                         if (sf_spec->sfs_type ==
 634                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 635                                 port1 = gen_spec->efs_loc_port;
 636                                 port2 = rport;
 637                         } else {
 638                                 port1 = rport;
 639                                 port2 = gen_spec->efs_loc_port;
 640                         }
 641                 }
 642                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 643                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 644                 sf_spec->sfs_dword[2] = host2;
 645                 break;
 646         }
 647
 648         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 649                 is_full = B_TRUE;
 650                 /* Fall through */
 651         case EFX_FILTER_MATCH_LOC_MAC:
 652                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 653                         sf_spec->sfs_type = (is_full ?
 654                             EFX_SIENA_FILTER_TX_MAC_FULL :
 655                             EFX_SIENA_FILTER_TX_MAC_WILD);
 656                 } else {
 657                         sf_spec->sfs_type = (is_full ?
 658                             EFX_SIENA_FILTER_RX_MAC_FULL :
 659                             EFX_SIENA_FILTER_RX_MAC_WILD);
 660                 }
 661                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 662                 sf_spec->sfs_dword[1] =
 663                     gen_spec->efs_loc_mac[2] << 24 |
 664                     gen_spec->efs_loc_mac[3] << 16 |
 665                     gen_spec->efs_loc_mac[4] <<  8 |
 666                     gen_spec->efs_loc_mac[5];
 667                 sf_spec->sfs_dword[2] =
 668                     gen_spec->efs_loc_mac[0] << 8 |
 669                     gen_spec->efs_loc_mac[1];
 670                 break;
 671
 672         default:
 673                 EFSYS_ASSERT(B_FALSE);
 674                 rc = ENOTSUP;
 675                 goto fail5;
 676         }
 677
 678         return (0);
 679
 680 fail5:
 681         EFSYS_PROBE(fail5);
 682 fail4:
 683         EFSYS_PROBE(fail4);
 684 fail3:
 685         EFSYS_PROBE(fail3);
 686 fail2:
 687         EFSYS_PROBE(fail2);
 688 fail1:
 689         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 690
 691         return (rc);
 692 }
 693
 694 /*
 695  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 696  * key derived from the n-tuple.
 697  */
 698 static                  uint16_t
 699 siena_filter_tbl_hash(
 700         __in            uint32_t key)
 701 {
 702         uint16_t tmp;
 703
 704         /* First 16 rounds */
 705         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 706         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 707         tmp = tmp ^ tmp >> 9;
 708
 709         /* Last 16 rounds */
 710         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 711         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 712         tmp = tmp ^ tmp >> 9;
 713
 714         return (tmp);
 715 }
 716
 717 /*
 718  * To allow for hash collisions, filter search continues at these
 719  * increments from the first possible entry selected by the hash.
 720  */
 721 static                  uint16_t
 722 siena_filter_tbl_increment(
 723         __in            uint32_t key)
 724 {
 725         return ((uint16_t)(key * 2 - 1));
 726 }
 727
 728 static  __checkReturn   boolean_t
 729 siena_filter_test_used(
 730         __in            siena_filter_tbl_t *sftp,
 731         __in            unsigned int index)
 732 {
 733         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 734         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 735 }
 736
 737 static                  void
 738 siena_filter_set_used(
 739         __in            siena_filter_tbl_t *sftp,
 740         __in            unsigned int index)
 741 {
 742         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 743         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 744         ++sftp->sft_used;
 745 }
 746
 747 static                  void
 748 siena_filter_clear_used(
 749         __in            siena_filter_tbl_t *sftp,
 750         __in            unsigned int index)
 751 {
 752         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 753         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 754
 755         --sftp->sft_used;
 756         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 757 }
 758
 759
 760 static                  siena_filter_tbl_id_t
 761 siena_filter_tbl_id(
 762         __in            siena_filter_type_t type)
 763 {
 764         siena_filter_tbl_id_t tbl_id;
 765
 766         switch (type) {
 767         case EFX_SIENA_FILTER_RX_TCP_FULL:
 768         case EFX_SIENA_FILTER_RX_TCP_WILD:
 769         case EFX_SIENA_FILTER_RX_UDP_FULL:
 770         case EFX_SIENA_FILTER_RX_UDP_WILD:
 771                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 772                 break;
 773
 774         case EFX_SIENA_FILTER_RX_MAC_FULL:
 775         case EFX_SIENA_FILTER_RX_MAC_WILD:
 776                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 777                 break;
 778
 779         case EFX_SIENA_FILTER_TX_TCP_FULL:
 780         case EFX_SIENA_FILTER_TX_TCP_WILD:
 781         case EFX_SIENA_FILTER_TX_UDP_FULL:
 782         case EFX_SIENA_FILTER_TX_UDP_WILD:
 783                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 784                 break;
 785
 786         case EFX_SIENA_FILTER_TX_MAC_FULL:
 787         case EFX_SIENA_FILTER_TX_MAC_WILD:
 788                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 789                 break;
 790
 791         default:
 792                 EFSYS_ASSERT(B_FALSE);
 793                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 794                 break;
 795         }
 796         return (tbl_id);
 797 }
 798
 799 static                  void
 800 siena_filter_reset_search_depth(
 801         __inout         siena_filter_t *sfp,
 802         __in            siena_filter_tbl_id_t tbl_id)
 803 {
 804         switch (tbl_id) {
 805         case EFX_SIENA_FILTER_TBL_RX_IP:
 806                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 807                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 808                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 809                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 810                 break;
 811
 812         case EFX_SIENA_FILTER_TBL_RX_MAC:
 813                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 814                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 815                 break;
 816
 817         case EFX_SIENA_FILTER_TBL_TX_IP:
 818                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 819                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 820                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 821                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 822                 break;
 823
 824         case EFX_SIENA_FILTER_TBL_TX_MAC:
 825                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 826                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 827                 break;
 828
 829         default:
 830                 EFSYS_ASSERT(B_FALSE);
 831                 break;
 832         }
 833 }
 834
 835 static                  void
 836 siena_filter_push_rx_limits(
 837         __in            efx_nic_t *enp)
 838 {
 839         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 840         efx_oword_t oword;
 841
 842         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 843
 844         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 845             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 846             FILTER_CTL_SRCH_FUDGE_FULL);
 847         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 848             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 849             FILTER_CTL_SRCH_FUDGE_WILD);
 850         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 851             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 852             FILTER_CTL_SRCH_FUDGE_FULL);
 853         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 854             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 855             FILTER_CTL_SRCH_FUDGE_WILD);
 856
 857         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 858                 EFX_SET_OWORD_FIELD(oword,
 859                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 860                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 861                     FILTER_CTL_SRCH_FUDGE_FULL);
 862                 EFX_SET_OWORD_FIELD(oword,
 863                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 864                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 865                     FILTER_CTL_SRCH_FUDGE_WILD);
 866         }
 867
 868         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 869 }
 870
 871 static                  void
 872 siena_filter_push_tx_limits(
 873         __in            efx_nic_t *enp)
 874 {
 875         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 876         efx_oword_t oword;
 877
 878         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 879
 880         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 881                 EFX_SET_OWORD_FIELD(oword,
 882                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 883                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 884                     FILTER_CTL_SRCH_FUDGE_FULL);
 885                 EFX_SET_OWORD_FIELD(oword,
 886                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 887                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 888                     FILTER_CTL_SRCH_FUDGE_WILD);
 889                 EFX_SET_OWORD_FIELD(oword,
 890                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 891                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 892                     FILTER_CTL_SRCH_FUDGE_FULL);
 893                 EFX_SET_OWORD_FIELD(oword,
 894                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 895                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 896                     FILTER_CTL_SRCH_FUDGE_WILD);
 897         }
 898
 899         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
 900                 EFX_SET_OWORD_FIELD(
 901                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 902                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
 903                         FILTER_CTL_SRCH_FUDGE_FULL);
 904                 EFX_SET_OWORD_FIELD(
 905                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 906                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
 907                         FILTER_CTL_SRCH_FUDGE_WILD);
 908         }
 909
 910         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 911 }
 912
 913 /* Build a filter entry and return its n-tuple key. */
 914 static  __checkReturn   uint32_t
 915 siena_filter_build(
 916         __out           efx_oword_t *filter,
 917         __in            siena_filter_spec_t *spec)
 918 {
 919         uint32_t dword3;
 920         uint32_t key;
 921         uint8_t  type  = spec->sfs_type;
 922         uint32_t flags = spec->sfs_flags;
 923
 924         switch (siena_filter_tbl_id(type)) {
 925         case EFX_SIENA_FILTER_TBL_RX_IP: {
 926                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
 927                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
 928                 EFX_POPULATE_OWORD_7(*filter,
 929                     FRF_BZ_RSS_EN,
 930                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 931                     FRF_BZ_SCATTER_EN,
 932                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 933                     FRF_AZ_TCP_UDP, is_udp,
 934                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
 935                     EFX_DWORD_2, spec->sfs_dword[2],
 936                     EFX_DWORD_1, spec->sfs_dword[1],
 937                     EFX_DWORD_0, spec->sfs_dword[0]);
 938                 dword3 = is_udp;
 939                 break;
 940         }
 941
 942         case EFX_SIENA_FILTER_TBL_RX_MAC: {
 943                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
 944                 EFX_POPULATE_OWORD_7(*filter,
 945                     FRF_CZ_RMFT_RSS_EN,
 946                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 947                     FRF_CZ_RMFT_SCATTER_EN,
 948                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 949                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
 950                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 951                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
 952                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
 953                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
 954                 dword3 = is_wild;
 955                 break;
 956         }
 957
 958         case EFX_SIENA_FILTER_TBL_TX_IP: {
 959                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
 960                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
 961                 EFX_POPULATE_OWORD_5(*filter,
 962                     FRF_CZ_TIFT_TCP_UDP, is_udp,
 963                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
 964                     EFX_DWORD_2, spec->sfs_dword[2],
 965                     EFX_DWORD_1, spec->sfs_dword[1],
 966                     EFX_DWORD_0, spec->sfs_dword[0]);
 967                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
 968                 break;
 969         }
 970
 971         case EFX_SIENA_FILTER_TBL_TX_MAC: {
 972                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
 973                 EFX_POPULATE_OWORD_5(*filter,
 974                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
 975                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 976                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
 977                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
 978                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
 979                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
 980                 break;
 981         }
 982
 983         default:
 984                 EFSYS_ASSERT(B_FALSE);
 985                 return (0);
 986         }
 987
 988         key =
 989             spec->sfs_dword[0] ^
 990             spec->sfs_dword[1] ^
 991             spec->sfs_dword[2] ^
 992             dword3;
 993
 994         return (key);
 995 }
 996
 997 static  __checkReturn           efx_rc_t
 998 siena_filter_push_entry(
 999         __inout                 efx_nic_t *enp,
1000         __in                    siena_filter_type_t type,
1001         __in                    int index,
1002         __in                    efx_oword_t *eop)
1003 {
1004         efx_rc_t rc;
1005
1006         switch (type) {
1007         case EFX_SIENA_FILTER_RX_TCP_FULL:
1008         case EFX_SIENA_FILTER_RX_TCP_WILD:
1009         case EFX_SIENA_FILTER_RX_UDP_FULL:
1010         case EFX_SIENA_FILTER_RX_UDP_WILD:
1011                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1012                     eop, B_TRUE);
1013                 break;
1014
1015         case EFX_SIENA_FILTER_RX_MAC_FULL:
1016         case EFX_SIENA_FILTER_RX_MAC_WILD:
1017                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1018                     eop, B_TRUE);
1019                 break;
1020
1021         case EFX_SIENA_FILTER_TX_TCP_FULL:
1022         case EFX_SIENA_FILTER_TX_TCP_WILD:
1023         case EFX_SIENA_FILTER_TX_UDP_FULL:
1024         case EFX_SIENA_FILTER_TX_UDP_WILD:
1025                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1026                     eop, B_TRUE);
1027                 break;
1028
1029         case EFX_SIENA_FILTER_TX_MAC_FULL:
1030         case EFX_SIENA_FILTER_TX_MAC_WILD:
1031                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1032                     eop, B_TRUE);
1033                 break;
1034
1035         default:
1036                 EFSYS_ASSERT(B_FALSE);
1037                 rc = ENOTSUP;
1038                 goto fail1;
1039         }
1040         return (0);
1041
1042 fail1:
1043         return (rc);
1044 }
1045
1046
1047 static  __checkReturn   boolean_t
1048 siena_filter_equal(
1049         __in            const siena_filter_spec_t *left,
1050         __in            const siena_filter_spec_t *right)
1051 {
1052         siena_filter_tbl_id_t tbl_id;
1053
1054         tbl_id = siena_filter_tbl_id(left->sfs_type);
1055
1056
1057         if (left->sfs_type != right->sfs_type)
1058                 return (B_FALSE);
1059
1060         if (memcmp(left->sfs_dword, right->sfs_dword,
1061                 sizeof (left->sfs_dword)))
1062                 return (B_FALSE);
1063
1064         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1065                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1066             left->sfs_dmaq_id != right->sfs_dmaq_id)
1067                 return (B_FALSE);
1068
1069         return (B_TRUE);
1070 }
1071
1072 static  __checkReturn   efx_rc_t
1073 siena_filter_search(
1074         __in            siena_filter_tbl_t *sftp,
1075         __in            siena_filter_spec_t *spec,
1076         __in            uint32_t key,
1077         __in            boolean_t for_insert,
1078         __out           int *filter_index,
1079         __out           unsigned int *depth_required)
1080 {
1081         unsigned int hash, incr, filter_idx, depth;
1082
1083         hash = siena_filter_tbl_hash(key);
1084         incr = siena_filter_tbl_increment(key);
1085
1086         filter_idx = hash & (sftp->sft_size - 1);
1087         depth = 1;
1088
1089         for (;;) {
1090                 /*
1091                  * Return success if entry is used and matches this spec
1092                  * or entry is unused and we are trying to insert.
1093                  */
1094                 if (siena_filter_test_used(sftp, filter_idx) ?
1095                     siena_filter_equal(spec,
1096                     &sftp->sft_spec[filter_idx]) :
1097                     for_insert) {
1098                         *filter_index = filter_idx;
1099                         *depth_required = depth;
1100                         return (0);
1101                 }
1102
1103                 /* Return failure if we reached the maximum search depth */
1104                 if (depth == FILTER_CTL_SRCH_MAX)
1105                         return (for_insert ? EBUSY : ENOENT);
1106
1107                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1108                 ++depth;
1109         }
1110 }
1111
1112 static                  void
1113 siena_filter_clear_entry(
1114         __in            efx_nic_t *enp,
1115         __in            siena_filter_tbl_t *sftp,
1116         __in            int index)
1117 {
1118         efx_oword_t filter;
1119
1120         if (siena_filter_test_used(sftp, index)) {
1121                 siena_filter_clear_used(sftp, index);
1122
1123                 EFX_ZERO_OWORD(filter);
1124                 siena_filter_push_entry(enp,
1125                     sftp->sft_spec[index].sfs_type,
1126                     index, &filter);
1127
1128                 memset(&sftp->sft_spec[index],
1129                     0, sizeof (sftp->sft_spec[0]));
1130         }
1131 }
1132
1133                         void
1134 siena_filter_tbl_clear(
1135         __in            efx_nic_t *enp,
1136         __in            siena_filter_tbl_id_t tbl_id)
1137 {
1138         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1139         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1140         int index;
1141         efsys_lock_state_t state;
1142
1143         EFSYS_LOCK(enp->en_eslp, state);
1144
1145         for (index = 0; index < sftp->sft_size; ++index) {
1146                 siena_filter_clear_entry(enp, sftp, index);
1147         }
1148
1149         if (sftp->sft_used == 0)
1150                 siena_filter_reset_search_depth(sfp, tbl_id);
1151
1152         EFSYS_UNLOCK(enp->en_eslp, state);
1153 }
1154
1155 static  __checkReturn   efx_rc_t
1156 siena_filter_init(
1157         __in            efx_nic_t *enp)
1158 {
1159         siena_filter_t *sfp;
1160         siena_filter_tbl_t *sftp;
1161         int tbl_id;
1162         efx_rc_t rc;
1163
1164         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1165
1166         if (!sfp) {
1167                 rc = ENOMEM;
1168                 goto fail1;
1169         }
1170
1171         enp->en_filter.ef_siena_filter = sfp;
1172
1173         switch (enp->en_family) {
1174         case EFX_FAMILY_SIENA:
1175                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1176                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1177
1178                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1179                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1180
1181                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1182                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1183
1184                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1185                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1186                 break;
1187
1188         default:
1189                 rc = ENOTSUP;
1190                 goto fail2;
1191         }
1192
1193         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1194                 unsigned int bitmap_size;
1195
1196                 sftp = &sfp->sf_tbl[tbl_id];
1197                 if (sftp->sft_size == 0)
1198                         continue;
1199
1200                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1201                     sizeof (uint32_t));
1202                 bitmap_size =
1203                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1204
1205                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1206                 if (!sftp->sft_bitmap) {
1207                         rc = ENOMEM;
1208                         goto fail3;
1209                 }
1210
1211                 EFSYS_KMEM_ALLOC(enp->en_esip,
1212                     sftp->sft_size * sizeof (*sftp->sft_spec),
1213                     sftp->sft_spec);
1214                 if (!sftp->sft_spec) {
1215                         rc = ENOMEM;
1216                         goto fail4;
1217                 }
1218                 memset(sftp->sft_spec, 0,
1219                     sftp->sft_size * sizeof (*sftp->sft_spec));
1220         }
1221
1222         return (0);
1223
1224 fail4:
1225         EFSYS_PROBE(fail4);
1226
1227 fail3:
1228         EFSYS_PROBE(fail3);
1229
1230 fail2:
1231         EFSYS_PROBE(fail2);
1232         siena_filter_fini(enp);
1233
1234 fail1:
1235         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1236         return (rc);
1237 }
1238
1239 static                  void
1240 siena_filter_fini(
1241         __in            efx_nic_t *enp)
1242 {
1243         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1244         siena_filter_tbl_id_t tbl_id;
1245
1246         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1247         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1248
1249         if (sfp == NULL)
1250                 return;
1251
1252         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1253                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1254                 unsigned int bitmap_size;
1255
1256                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1257                     sizeof (uint32_t));
1258                 bitmap_size =
1259                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1260
1261                 if (sftp->sft_bitmap != NULL) {
1262                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1263                             sftp->sft_bitmap);
1264                         sftp->sft_bitmap = NULL;
1265                 }
1266
1267                 if (sftp->sft_spec != NULL) {
1268                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1269                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1270                         sftp->sft_spec = NULL;
1271                 }
1272         }
1273
1274         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1275             enp->en_filter.ef_siena_filter);
1276 }
1277
1278 /* Restore filter state after a reset */
1279 static  __checkReturn   efx_rc_t
1280 siena_filter_restore(
1281         __in            efx_nic_t *enp)
1282 {
1283         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1284         siena_filter_tbl_id_t tbl_id;
1285         siena_filter_tbl_t *sftp;
1286         siena_filter_spec_t *spec;
1287         efx_oword_t filter;
1288         int filter_idx;
1289         efsys_lock_state_t state;
1290         uint32_t key;
1291         efx_rc_t rc;
1292
1293         EFSYS_LOCK(enp->en_eslp, state);
1294
1295         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1296                 sftp = &sfp->sf_tbl[tbl_id];
1297                 for (filter_idx = 0;
1298                         filter_idx < sftp->sft_size;
1299                         filter_idx++) {
1300                         if (!siena_filter_test_used(sftp, filter_idx))
1301                                 continue;
1302
1303                         spec = &sftp->sft_spec[filter_idx];
1304                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1305                                 rc = EINVAL;
1306                                 goto fail1;
1307                         }
1308                         if ((rc = siena_filter_push_entry(enp,
1309                                     spec->sfs_type, filter_idx, &filter)) != 0)
1310                                 goto fail2;
1311                 }
1312         }
1313
1314         siena_filter_push_rx_limits(enp);
1315         siena_filter_push_tx_limits(enp);
1316
1317         EFSYS_UNLOCK(enp->en_eslp, state);
1318
1319         return (0);
1320
1321 fail2:
1322         EFSYS_PROBE(fail2);
1323
1324 fail1:
1325         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1326
1327         EFSYS_UNLOCK(enp->en_eslp, state);
1328
1329         return (rc);
1330 }
1331
1332 static   __checkReturn  efx_rc_t
1333 siena_filter_add(
1334         __in            efx_nic_t *enp,
1335         __inout         efx_filter_spec_t *spec,
1336         __in            boolean_t may_replace)
1337 {
1338         efx_rc_t rc;
1339         siena_filter_spec_t sf_spec;
1340         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1341         siena_filter_tbl_id_t tbl_id;
1342         siena_filter_tbl_t *sftp;
1343         siena_filter_spec_t *saved_sf_spec;
1344         efx_oword_t filter;
1345         int filter_idx;
1346         unsigned int depth;
1347         efsys_lock_state_t state;
1348         uint32_t key;
1349
1350
1351         EFSYS_ASSERT3P(spec, !=, NULL);
1352
1353         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1354                 goto fail1;
1355
1356         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1357         sftp = &sfp->sf_tbl[tbl_id];
1358
1359         if (sftp->sft_size == 0) {
1360                 rc = EINVAL;
1361                 goto fail2;
1362         }
1363
1364         key = siena_filter_build(&filter, &sf_spec);
1365
1366         EFSYS_LOCK(enp->en_eslp, state);
1367
1368         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1369             &filter_idx, &depth);
1370         if (rc != 0)
1371                 goto fail3;
1372
1373         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1374         saved_sf_spec = &sftp->sft_spec[filter_idx];
1375
1376         if (siena_filter_test_used(sftp, filter_idx)) {
1377                 if (may_replace == B_FALSE) {
1378                         rc = EEXIST;
1379                         goto fail4;
1380                 }
1381         }
1382         siena_filter_set_used(sftp, filter_idx);
1383         *saved_sf_spec = sf_spec;
1384
1385         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1386                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1387                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1388                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1389                         siena_filter_push_tx_limits(enp);
1390                 else
1391                         siena_filter_push_rx_limits(enp);
1392         }
1393
1394         siena_filter_push_entry(enp, sf_spec.sfs_type,
1395             filter_idx, &filter);
1396
1397         EFSYS_UNLOCK(enp->en_eslp, state);
1398         return (0);
1399
1400 fail4:
1401         EFSYS_PROBE(fail4);
1402
1403 fail3:
1404         EFSYS_UNLOCK(enp->en_eslp, state);
1405         EFSYS_PROBE(fail3);
1406
1407 fail2:
1408         EFSYS_PROBE(fail2);
1409
1410 fail1:
1411         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1412         return (rc);
1413 }
1414
1415 static   __checkReturn  efx_rc_t
1416 siena_filter_delete(
1417         __in            efx_nic_t *enp,
1418         __inout         efx_filter_spec_t *spec)
1419 {
1420         efx_rc_t rc;
1421         siena_filter_spec_t sf_spec;
1422         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1423         siena_filter_tbl_id_t tbl_id;
1424         siena_filter_tbl_t *sftp;
1425         efx_oword_t filter;
1426         int filter_idx;
1427         unsigned int depth;
1428         efsys_lock_state_t state;
1429         uint32_t key;
1430
1431         EFSYS_ASSERT3P(spec, !=, NULL);
1432
1433         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1434                 goto fail1;
1435
1436         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1437         sftp = &sfp->sf_tbl[tbl_id];
1438
1439         key = siena_filter_build(&filter, &sf_spec);
1440
1441         EFSYS_LOCK(enp->en_eslp, state);
1442
1443         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1444             &filter_idx, &depth);
1445         if (rc != 0)
1446                 goto fail2;
1447
1448         siena_filter_clear_entry(enp, sftp, filter_idx);
1449         if (sftp->sft_used == 0)
1450                 siena_filter_reset_search_depth(sfp, tbl_id);
1451
1452         EFSYS_UNLOCK(enp->en_eslp, state);
1453         return (0);
1454
1455 fail2:
1456         EFSYS_UNLOCK(enp->en_eslp, state);
1457         EFSYS_PROBE(fail2);
1458
1459 fail1:
1460         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1461         return (rc);
1462 }
1463
1464 #define SIENA_MAX_SUPPORTED_MATCHES 4
1465
1466 static  __checkReturn   efx_rc_t
1467 siena_filter_supported_filters(
1468         __in                            efx_nic_t *enp,
1469         __out_ecount(buffer_length)     uint32_t *buffer,
1470         __in                            size_t buffer_length,
1471         __out                           size_t *list_lengthp)
1472 {
1473         uint32_t index = 0;
1474         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1475         size_t list_length;
1476         efx_rc_t rc;
1477
1478         rx_matches[index++] =
1479             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1480             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1481             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1482
1483         rx_matches[index++] =
1484             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1485             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1486
1487         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1488                 rx_matches[index++] =
1489                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1490
1491                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1492         }
1493
1494         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1495         list_length = index;
1496
1497         *list_lengthp = list_length;
1498
1499         if (buffer_length < list_length) {
1500                 rc = ENOSPC;
1501                 goto fail1;
1502         }
1503
1504         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1505
1506         return (0);
1507
1508 fail1:
1509         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1510
1511         return (rc);
1512 }
1513
1514 #undef MAX_SUPPORTED
1515
1516 #endif /* EFSYS_OPT_SIENA */
1517
1518 #endif /* EFSYS_OPT_FILTER */