sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include "efx.h"
  37 #include "efx_impl.h"
  38
  39
  40 #if EFSYS_OPT_FILTER
  41
  42 #if EFSYS_OPT_SIENA
  43
  44 static  __checkReturn   efx_rc_t
  45 siena_filter_init(
  46         __in            efx_nic_t *enp);
  47
  48 static                  void
  49 siena_filter_fini(
  50         __in            efx_nic_t *enp);
  51
  52 static  __checkReturn   efx_rc_t
  53 siena_filter_restore(
  54         __in            efx_nic_t *enp);
  55
  56 static  __checkReturn   efx_rc_t
  57 siena_filter_add(
  58         __in            efx_nic_t *enp,
  59         __inout         efx_filter_spec_t *spec,
  60         __in            boolean_t may_replace);
  61
  62 static  __checkReturn   efx_rc_t
  63 siena_filter_delete(
  64         __in            efx_nic_t *enp,
  65         __inout         efx_filter_spec_t *spec);
  66
  67 static  __checkReturn   efx_rc_t
  68 siena_filter_supported_filters(
  69         __in                            efx_nic_t *enp,
  70         __out_ecount(buffer_length)     uint32_t *buffer,
  71         __in                            size_t buffer_length,
  72         __out                           size_t *list_lengthp);
  73
  74 #endif /* EFSYS_OPT_SIENA */
  75
  76 #if EFSYS_OPT_SIENA
  77 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  78         siena_filter_init,              /* efo_init */
  79         siena_filter_fini,              /* efo_fini */
  80         siena_filter_restore,           /* efo_restore */
  81         siena_filter_add,               /* efo_add */
  82         siena_filter_delete,            /* efo_delete */
  83         siena_filter_supported_filters, /* efo_supported_filters */
  84         NULL,                           /* efo_reconfigure */
  85 };
  86 #endif /* EFSYS_OPT_SIENA */
  87
  88 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  89 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  90         ef10_filter_init,               /* efo_init */
  91         ef10_filter_fini,               /* efo_fini */
  92         ef10_filter_restore,            /* efo_restore */
  93         ef10_filter_add,                /* efo_add */
  94         ef10_filter_delete,             /* efo_delete */
  95         ef10_filter_supported_filters,  /* efo_supported_filters */
  96         ef10_filter_reconfigure,        /* efo_reconfigure */
  97 };
  98 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
  99
 100         __checkReturn   efx_rc_t
 101 efx_filter_insert(
 102         __in            efx_nic_t *enp,
 103         __inout         efx_filter_spec_t *spec)
 104 {
 105         const efx_filter_ops_t *efop = enp->en_efop;
 106
 107         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 108         EFSYS_ASSERT3P(spec, !=, NULL);
 109         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 110
 111         return (efop->efo_add(enp, spec, B_FALSE));
 112 }
 113
 114         __checkReturn   efx_rc_t
 115 efx_filter_remove(
 116         __in            efx_nic_t *enp,
 117         __inout         efx_filter_spec_t *spec)
 118 {
 119         const efx_filter_ops_t *efop = enp->en_efop;
 120
 121         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 122         EFSYS_ASSERT3P(spec, !=, NULL);
 123         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 124
 125         return (efop->efo_delete(enp, spec));
 126 }
 127
 128         __checkReturn   efx_rc_t
 129 efx_filter_restore(
 130         __in            efx_nic_t *enp)
 131 {
 132         efx_rc_t rc;
 133
 134         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 135
 136         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 137                 goto fail1;
 138
 139         return (0);
 140
 141 fail1:
 142         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 143
 144         return (rc);
 145 }
 146
 147         __checkReturn   efx_rc_t
 148 efx_filter_init(
 149         __in            efx_nic_t *enp)
 150 {
 151         const efx_filter_ops_t *efop;
 152         efx_rc_t rc;
 153
 154         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 155         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 156         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 157
 158         switch (enp->en_family) {
 159 #if EFSYS_OPT_SIENA
 160         case EFX_FAMILY_SIENA:
 161                 efop = &__efx_filter_siena_ops;
 162                 break;
 163 #endif /* EFSYS_OPT_SIENA */
 164
 165 #if EFSYS_OPT_HUNTINGTON
 166         case EFX_FAMILY_HUNTINGTON:
 167                 efop = &__efx_filter_ef10_ops;
 168                 break;
 169 #endif /* EFSYS_OPT_HUNTINGTON */
 170
 171 #if EFSYS_OPT_MEDFORD
 172         case EFX_FAMILY_MEDFORD:
 173                 efop = &__efx_filter_ef10_ops;
 174                 break;
 175 #endif /* EFSYS_OPT_MEDFORD */
 176
 177 #if EFSYS_OPT_MEDFORD2
 178         case EFX_FAMILY_MEDFORD2:
 179                 efop = &__efx_filter_ef10_ops;
 180                 break;
 181 #endif /* EFSYS_OPT_MEDFORD2 */
 182
 183         default:
 184                 EFSYS_ASSERT(0);
 185                 rc = ENOTSUP;
 186                 goto fail1;
 187         }
 188
 189         if ((rc = efop->efo_init(enp)) != 0)
 190                 goto fail2;
 191
 192         enp->en_efop = efop;
 193         enp->en_mod_flags |= EFX_MOD_FILTER;
 194         return (0);
 195
 196 fail2:
 197         EFSYS_PROBE(fail2);
 198 fail1:
 199         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 200
 201         enp->en_efop = NULL;
 202         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 203         return (rc);
 204 }
 205
 206                         void
 207 efx_filter_fini(
 208         __in            efx_nic_t *enp)
 209 {
 210         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 211         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 212         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 213
 214         enp->en_efop->efo_fini(enp);
 215
 216         enp->en_efop = NULL;
 217         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 218 }
 219
 220 /*
 221  * Query the possible combinations of match flags which can be filtered on.
 222  * These are returned as a list, of which each 32 bit element is a bitmask
 223  * formed of EFX_FILTER_MATCH flags.
 224  *
 225  * The combinations are ordered in priority from highest to lowest.
 226  *
 227  * If the provided buffer is too short to hold the list, the call with fail with
 228  * ENOSPC and *list_lengthp will be set to the buffer length required.
 229  */
 230         __checkReturn   efx_rc_t
 231 efx_filter_supported_filters(
 232         __in                            efx_nic_t *enp,
 233         __out_ecount(buffer_length)     uint32_t *buffer,
 234         __in                            size_t buffer_length,
 235         __out                           size_t *list_lengthp)
 236 {
 237         efx_rc_t rc;
 238
 239         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 240         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 241         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 242         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 243
 244         if (buffer == NULL) {
 245                 rc = EINVAL;
 246                 goto fail1;
 247         }
 248
 249         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 250                                                     list_lengthp);
 251         if (rc != 0)
 252                 goto fail2;
 253
 254         return (0);
 255
 256 fail2:
 257         EFSYS_PROBE(fail2);
 258 fail1:
 259         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 260
 261         return (rc);
 262 }
 263
 264         __checkReturn   efx_rc_t
 265 efx_filter_reconfigure(
 266         __in                            efx_nic_t *enp,
 267         __in_ecount(6)                  uint8_t const *mac_addr,
 268         __in                            boolean_t all_unicst,
 269         __in                            boolean_t mulcst,
 270         __in                            boolean_t all_mulcst,
 271         __in                            boolean_t brdcst,
 272         __in_ecount(6*count)            uint8_t const *addrs,
 273         __in                            uint32_t count)
 274 {
 275         efx_rc_t rc;
 276
 277         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 278         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 279         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 280
 281         if (enp->en_efop->efo_reconfigure != NULL) {
 282                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 283                                                         all_unicst, mulcst,
 284                                                         all_mulcst, brdcst,
 285                                                         addrs, count)) != 0)
 286                         goto fail1;
 287         }
 288
 289         return (0);
 290
 291 fail1:
 292         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 293
 294         return (rc);
 295 }
 296
 297                 void
 298 efx_filter_spec_init_rx(
 299         __out           efx_filter_spec_t *spec,
 300         __in            efx_filter_priority_t priority,
 301         __in            efx_filter_flags_t flags,
 302         __in            efx_rxq_t *erp)
 303 {
 304         EFSYS_ASSERT3P(spec, !=, NULL);
 305         EFSYS_ASSERT3P(erp, !=, NULL);
 306         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 307                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 308
 309         memset(spec, 0, sizeof (*spec));
 310         spec->efs_priority = priority;
 311         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 312         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 313         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 314 }
 315
 316                 void
 317 efx_filter_spec_init_tx(
 318         __out           efx_filter_spec_t *spec,
 319         __in            efx_txq_t *etp)
 320 {
 321         EFSYS_ASSERT3P(spec, !=, NULL);
 322         EFSYS_ASSERT3P(etp, !=, NULL);
 323
 324         memset(spec, 0, sizeof (*spec));
 325         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 326         spec->efs_flags = EFX_FILTER_FLAG_TX;
 327         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 328 }
 329
 330
 331 /*
 332  *  Specify IPv4 host, transport protocol and port in a filter specification
 333  */
 334 __checkReturn           efx_rc_t
 335 efx_filter_spec_set_ipv4_local(
 336         __inout         efx_filter_spec_t *spec,
 337         __in            uint8_t proto,
 338         __in            uint32_t host,
 339         __in            uint16_t port)
 340 {
 341         EFSYS_ASSERT3P(spec, !=, NULL);
 342
 343         spec->efs_match_flags |=
 344                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 345                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 346         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 347         spec->efs_ip_proto = proto;
 348         spec->efs_loc_host.eo_u32[0] = host;
 349         spec->efs_loc_port = port;
 350         return (0);
 351 }
 352
 353 /*
 354  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 355  */
 356 __checkReturn           efx_rc_t
 357 efx_filter_spec_set_ipv4_full(
 358         __inout         efx_filter_spec_t *spec,
 359         __in            uint8_t proto,
 360         __in            uint32_t lhost,
 361         __in            uint16_t lport,
 362         __in            uint32_t rhost,
 363         __in            uint16_t rport)
 364 {
 365         EFSYS_ASSERT3P(spec, !=, NULL);
 366
 367         spec->efs_match_flags |=
 368                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 369                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 370                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 371         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 372         spec->efs_ip_proto = proto;
 373         spec->efs_loc_host.eo_u32[0] = lhost;
 374         spec->efs_loc_port = lport;
 375         spec->efs_rem_host.eo_u32[0] = rhost;
 376         spec->efs_rem_port = rport;
 377         return (0);
 378 }
 379
 380 /*
 381  * Specify local Ethernet address and/or VID in filter specification
 382  */
 383 __checkReturn           efx_rc_t
 384 efx_filter_spec_set_eth_local(
 385         __inout         efx_filter_spec_t *spec,
 386         __in            uint16_t vid,
 387         __in            const uint8_t *addr)
 388 {
 389         EFSYS_ASSERT3P(spec, !=, NULL);
 390         EFSYS_ASSERT3P(addr, !=, NULL);
 391
 392         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 393                 return (EINVAL);
 394
 395         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 396                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 397                 spec->efs_outer_vid = vid;
 398         }
 399         if (addr != NULL) {
 400                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 401                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 402         }
 403         return (0);
 404 }
 405
 406                         void
 407 efx_filter_spec_set_ether_type(
 408         __inout         efx_filter_spec_t *spec,
 409         __in            uint16_t ether_type)
 410 {
 411         EFSYS_ASSERT3P(spec, !=, NULL);
 412
 413         spec->efs_ether_type = ether_type;
 414         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 415 }
 416
 417 /*
 418  * Specify matching otherwise-unmatched unicast in a filter specification
 419  */
 420 __checkReturn           efx_rc_t
 421 efx_filter_spec_set_uc_def(
 422         __inout         efx_filter_spec_t *spec)
 423 {
 424         EFSYS_ASSERT3P(spec, !=, NULL);
 425
 426         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 427         return (0);
 428 }
 429
 430 /*
 431  * Specify matching otherwise-unmatched multicast in a filter specification
 432  */
 433 __checkReturn           efx_rc_t
 434 efx_filter_spec_set_mc_def(
 435         __inout         efx_filter_spec_t *spec)
 436 {
 437         EFSYS_ASSERT3P(spec, !=, NULL);
 438
 439         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 440         return (0);
 441 }
 442
 443
 444 __checkReturn           efx_rc_t
 445 efx_filter_spec_set_encap_type(
 446         __inout         efx_filter_spec_t *spec,
 447         __in            efx_tunnel_protocol_t encap_type,
 448         __in            efx_filter_inner_frame_match_t inner_frame_match)
 449 {
 450         uint32_t match_flags = 0;
 451         uint8_t ip_proto;
 452         efx_rc_t rc;
 453
 454         EFSYS_ASSERT3P(spec, !=, NULL);
 455
 456         switch (encap_type) {
 457         case EFX_TUNNEL_PROTOCOL_VXLAN:
 458         case EFX_TUNNEL_PROTOCOL_GENEVE:
 459                 ip_proto = EFX_IPPROTO_UDP;
 460                 break;
 461         case EFX_TUNNEL_PROTOCOL_NVGRE:
 462                 ip_proto = EFX_IPPROTO_GRE;
 463                 break;
 464         default:
 465                 EFSYS_ASSERT(0);
 466                 rc = EINVAL;
 467                 goto fail1;
 468         }
 469
 470         switch (inner_frame_match) {
 471         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 472                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 473                 break;
 474         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 475                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 476                 break;
 477         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 478                 /* This is for when specific inner frames are to be matched. */
 479                 break;
 480         default:
 481                 EFSYS_ASSERT(0);
 482                 rc = EINVAL;
 483                 goto fail2;
 484         }
 485
 486         spec->efs_encap_type = encap_type;
 487         spec->efs_ip_proto = ip_proto;
 488         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 489
 490         return (0);
 491
 492 fail2:
 493         EFSYS_PROBE(fail2);
 494 fail1:
 495         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 496
 497         return (rc);
 498 }
 499
 500 #if EFSYS_OPT_RX_SCALE
 501         __checkReturn   efx_rc_t
 502 efx_filter_spec_set_rss_context(
 503         __inout         efx_filter_spec_t *spec,
 504         __in            uint32_t rss_context)
 505 {
 506         efx_rc_t rc;
 507
 508         EFSYS_ASSERT3P(spec, !=, NULL);
 509
 510         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 511         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 512                 rc = EINVAL;
 513                 goto fail1;
 514         }
 515
 516         spec->efs_rss_context = rss_context;
 517
 518         return (0);
 519
 520 fail1:
 521         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 522
 523         return (rc);
 524 }
 525 #endif
 526
 527 #if EFSYS_OPT_SIENA
 528
 529 /*
 530  * "Fudge factors" - difference between programmed value and actual depth.
 531  * Due to pipelined implementation we need to program H/W with a value that
 532  * is larger than the hop limit we want.
 533  */
 534 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 535 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 536
 537 /*
 538  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 539  * We also need to avoid infinite loops in efx_filter_search() when the
 540  * table is full.
 541  */
 542 #define FILTER_CTL_SRCH_MAX 200
 543
 544 static  __checkReturn   efx_rc_t
 545 siena_filter_spec_from_gen_spec(
 546         __out           siena_filter_spec_t *sf_spec,
 547         __in            efx_filter_spec_t *gen_spec)
 548 {
 549         efx_rc_t rc;
 550         boolean_t is_full = B_FALSE;
 551
 552         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 553                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 554         else
 555                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 556
 557         /* Siena only has one RSS context */
 558         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 559             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 560                 rc = EINVAL;
 561                 goto fail1;
 562         }
 563
 564         sf_spec->sfs_flags = gen_spec->efs_flags;
 565         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 566
 567         switch (gen_spec->efs_match_flags) {
 568         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 569             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 570             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 571                 is_full = B_TRUE;
 572                 /* Fall through */
 573         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 574             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 575                 uint32_t rhost, host1, host2;
 576                 uint16_t rport, port1, port2;
 577
 578                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 579                         rc = ENOTSUP;
 580                         goto fail2;
 581                 }
 582                 if (gen_spec->efs_loc_port == 0 ||
 583                     (is_full && gen_spec->efs_rem_port == 0)) {
 584                         rc = EINVAL;
 585                         goto fail3;
 586                 }
 587                 switch (gen_spec->efs_ip_proto) {
 588                 case EFX_IPPROTO_TCP:
 589                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 590                                 sf_spec->sfs_type = (is_full ?
 591                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 592                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 593                         } else {
 594                                 sf_spec->sfs_type = (is_full ?
 595                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 596                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 597                         }
 598                         break;
 599                 case EFX_IPPROTO_UDP:
 600                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 601                                 sf_spec->sfs_type = (is_full ?
 602                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 603                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 604                         } else {
 605                                 sf_spec->sfs_type = (is_full ?
 606                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 607                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 608                         }
 609                         break;
 610                 default:
 611                         rc = ENOTSUP;
 612                         goto fail4;
 613                 }
 614                 /*
 615                  * The filter is constructed in terms of source and destination,
 616                  * with the odd wrinkle that the ports are swapped in a UDP
 617                  * wildcard filter. We need to convert from local and remote
 618                  * addresses (zero for a wildcard).
 619                  */
 620                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 621                 rport = is_full ? gen_spec->efs_rem_port : 0;
 622                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 623                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 624                         host2 = rhost;
 625                 } else {
 626                         host1 = rhost;
 627                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 628                 }
 629                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 630                         if (sf_spec->sfs_type ==
 631                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 632                                 port1 = rport;
 633                                 port2 = gen_spec->efs_loc_port;
 634                         } else {
 635                                 port1 = gen_spec->efs_loc_port;
 636                                 port2 = rport;
 637                         }
 638                 } else {
 639                         if (sf_spec->sfs_type ==
 640                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 641                                 port1 = gen_spec->efs_loc_port;
 642                                 port2 = rport;
 643                         } else {
 644                                 port1 = rport;
 645                                 port2 = gen_spec->efs_loc_port;
 646                         }
 647                 }
 648                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 649                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 650                 sf_spec->sfs_dword[2] = host2;
 651                 break;
 652         }
 653
 654         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 655                 is_full = B_TRUE;
 656                 /* Fall through */
 657         case EFX_FILTER_MATCH_LOC_MAC:
 658                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 659                         sf_spec->sfs_type = (is_full ?
 660                             EFX_SIENA_FILTER_TX_MAC_FULL :
 661                             EFX_SIENA_FILTER_TX_MAC_WILD);
 662                 } else {
 663                         sf_spec->sfs_type = (is_full ?
 664                             EFX_SIENA_FILTER_RX_MAC_FULL :
 665                             EFX_SIENA_FILTER_RX_MAC_WILD);
 666                 }
 667                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 668                 sf_spec->sfs_dword[1] =
 669                     gen_spec->efs_loc_mac[2] << 24 |
 670                     gen_spec->efs_loc_mac[3] << 16 |
 671                     gen_spec->efs_loc_mac[4] <<  8 |
 672                     gen_spec->efs_loc_mac[5];
 673                 sf_spec->sfs_dword[2] =
 674                     gen_spec->efs_loc_mac[0] << 8 |
 675                     gen_spec->efs_loc_mac[1];
 676                 break;
 677
 678         default:
 679                 EFSYS_ASSERT(B_FALSE);
 680                 rc = ENOTSUP;
 681                 goto fail5;
 682         }
 683
 684         return (0);
 685
 686 fail5:
 687         EFSYS_PROBE(fail5);
 688 fail4:
 689         EFSYS_PROBE(fail4);
 690 fail3:
 691         EFSYS_PROBE(fail3);
 692 fail2:
 693         EFSYS_PROBE(fail2);
 694 fail1:
 695         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 696
 697         return (rc);
 698 }
 699
 700 /*
 701  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 702  * key derived from the n-tuple.
 703  */
 704 static                  uint16_t
 705 siena_filter_tbl_hash(
 706         __in            uint32_t key)
 707 {
 708         uint16_t tmp;
 709
 710         /* First 16 rounds */
 711         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 712         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 713         tmp = tmp ^ tmp >> 9;
 714
 715         /* Last 16 rounds */
 716         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 717         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 718         tmp = tmp ^ tmp >> 9;
 719
 720         return (tmp);
 721 }
 722
 723 /*
 724  * To allow for hash collisions, filter search continues at these
 725  * increments from the first possible entry selected by the hash.
 726  */
 727 static                  uint16_t
 728 siena_filter_tbl_increment(
 729         __in            uint32_t key)
 730 {
 731         return ((uint16_t)(key * 2 - 1));
 732 }
 733
 734 static  __checkReturn   boolean_t
 735 siena_filter_test_used(
 736         __in            siena_filter_tbl_t *sftp,
 737         __in            unsigned int index)
 738 {
 739         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 740         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 741 }
 742
 743 static                  void
 744 siena_filter_set_used(
 745         __in            siena_filter_tbl_t *sftp,
 746         __in            unsigned int index)
 747 {
 748         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 749         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 750         ++sftp->sft_used;
 751 }
 752
 753 static                  void
 754 siena_filter_clear_used(
 755         __in            siena_filter_tbl_t *sftp,
 756         __in            unsigned int index)
 757 {
 758         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 759         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 760
 761         --sftp->sft_used;
 762         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 763 }
 764
 765
 766 static                  siena_filter_tbl_id_t
 767 siena_filter_tbl_id(
 768         __in            siena_filter_type_t type)
 769 {
 770         siena_filter_tbl_id_t tbl_id;
 771
 772         switch (type) {
 773         case EFX_SIENA_FILTER_RX_TCP_FULL:
 774         case EFX_SIENA_FILTER_RX_TCP_WILD:
 775         case EFX_SIENA_FILTER_RX_UDP_FULL:
 776         case EFX_SIENA_FILTER_RX_UDP_WILD:
 777                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 778                 break;
 779
 780         case EFX_SIENA_FILTER_RX_MAC_FULL:
 781         case EFX_SIENA_FILTER_RX_MAC_WILD:
 782                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 783                 break;
 784
 785         case EFX_SIENA_FILTER_TX_TCP_FULL:
 786         case EFX_SIENA_FILTER_TX_TCP_WILD:
 787         case EFX_SIENA_FILTER_TX_UDP_FULL:
 788         case EFX_SIENA_FILTER_TX_UDP_WILD:
 789                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 790                 break;
 791
 792         case EFX_SIENA_FILTER_TX_MAC_FULL:
 793         case EFX_SIENA_FILTER_TX_MAC_WILD:
 794                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 795                 break;
 796
 797         default:
 798                 EFSYS_ASSERT(B_FALSE);
 799                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 800                 break;
 801         }
 802         return (tbl_id);
 803 }
 804
 805 static                  void
 806 siena_filter_reset_search_depth(
 807         __inout         siena_filter_t *sfp,
 808         __in            siena_filter_tbl_id_t tbl_id)
 809 {
 810         switch (tbl_id) {
 811         case EFX_SIENA_FILTER_TBL_RX_IP:
 812                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 813                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 814                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 815                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 816                 break;
 817
 818         case EFX_SIENA_FILTER_TBL_RX_MAC:
 819                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 820                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 821                 break;
 822
 823         case EFX_SIENA_FILTER_TBL_TX_IP:
 824                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 825                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 826                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 827                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 828                 break;
 829
 830         case EFX_SIENA_FILTER_TBL_TX_MAC:
 831                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 832                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 833                 break;
 834
 835         default:
 836                 EFSYS_ASSERT(B_FALSE);
 837                 break;
 838         }
 839 }
 840
 841 static                  void
 842 siena_filter_push_rx_limits(
 843         __in            efx_nic_t *enp)
 844 {
 845         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 846         efx_oword_t oword;
 847
 848         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 849
 850         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 851             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 852             FILTER_CTL_SRCH_FUDGE_FULL);
 853         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 854             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 855             FILTER_CTL_SRCH_FUDGE_WILD);
 856         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 857             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 858             FILTER_CTL_SRCH_FUDGE_FULL);
 859         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 860             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 861             FILTER_CTL_SRCH_FUDGE_WILD);
 862
 863         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 864                 EFX_SET_OWORD_FIELD(oword,
 865                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 866                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 867                     FILTER_CTL_SRCH_FUDGE_FULL);
 868                 EFX_SET_OWORD_FIELD(oword,
 869                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 870                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 871                     FILTER_CTL_SRCH_FUDGE_WILD);
 872         }
 873
 874         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 875 }
 876
 877 static                  void
 878 siena_filter_push_tx_limits(
 879         __in            efx_nic_t *enp)
 880 {
 881         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 882         efx_oword_t oword;
 883
 884         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 885
 886         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 887                 EFX_SET_OWORD_FIELD(oword,
 888                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 889                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 890                     FILTER_CTL_SRCH_FUDGE_FULL);
 891                 EFX_SET_OWORD_FIELD(oword,
 892                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 893                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 894                     FILTER_CTL_SRCH_FUDGE_WILD);
 895                 EFX_SET_OWORD_FIELD(oword,
 896                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 897                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 898                     FILTER_CTL_SRCH_FUDGE_FULL);
 899                 EFX_SET_OWORD_FIELD(oword,
 900                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 901                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 902                     FILTER_CTL_SRCH_FUDGE_WILD);
 903         }
 904
 905         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
 906                 EFX_SET_OWORD_FIELD(
 907                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 908                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
 909                         FILTER_CTL_SRCH_FUDGE_FULL);
 910                 EFX_SET_OWORD_FIELD(
 911                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 912                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
 913                         FILTER_CTL_SRCH_FUDGE_WILD);
 914         }
 915
 916         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 917 }
 918
 919 /* Build a filter entry and return its n-tuple key. */
 920 static  __checkReturn   uint32_t
 921 siena_filter_build(
 922         __out           efx_oword_t *filter,
 923         __in            siena_filter_spec_t *spec)
 924 {
 925         uint32_t dword3;
 926         uint32_t key;
 927         uint8_t  type  = spec->sfs_type;
 928         uint32_t flags = spec->sfs_flags;
 929
 930         switch (siena_filter_tbl_id(type)) {
 931         case EFX_SIENA_FILTER_TBL_RX_IP: {
 932                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
 933                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
 934                 EFX_POPULATE_OWORD_7(*filter,
 935                     FRF_BZ_RSS_EN,
 936                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 937                     FRF_BZ_SCATTER_EN,
 938                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 939                     FRF_AZ_TCP_UDP, is_udp,
 940                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
 941                     EFX_DWORD_2, spec->sfs_dword[2],
 942                     EFX_DWORD_1, spec->sfs_dword[1],
 943                     EFX_DWORD_0, spec->sfs_dword[0]);
 944                 dword3 = is_udp;
 945                 break;
 946         }
 947
 948         case EFX_SIENA_FILTER_TBL_RX_MAC: {
 949                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
 950                 EFX_POPULATE_OWORD_7(*filter,
 951                     FRF_CZ_RMFT_RSS_EN,
 952                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 953                     FRF_CZ_RMFT_SCATTER_EN,
 954                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 955                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
 956                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 957                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
 958                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
 959                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
 960                 dword3 = is_wild;
 961                 break;
 962         }
 963
 964         case EFX_SIENA_FILTER_TBL_TX_IP: {
 965                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
 966                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
 967                 EFX_POPULATE_OWORD_5(*filter,
 968                     FRF_CZ_TIFT_TCP_UDP, is_udp,
 969                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
 970                     EFX_DWORD_2, spec->sfs_dword[2],
 971                     EFX_DWORD_1, spec->sfs_dword[1],
 972                     EFX_DWORD_0, spec->sfs_dword[0]);
 973                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
 974                 break;
 975         }
 976
 977         case EFX_SIENA_FILTER_TBL_TX_MAC: {
 978                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
 979                 EFX_POPULATE_OWORD_5(*filter,
 980                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
 981                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 982                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
 983                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
 984                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
 985                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
 986                 break;
 987         }
 988
 989         default:
 990                 EFSYS_ASSERT(B_FALSE);
 991                 return (0);
 992         }
 993
 994         key =
 995             spec->sfs_dword[0] ^
 996             spec->sfs_dword[1] ^
 997             spec->sfs_dword[2] ^
 998             dword3;
 999
1000         return (key);
1001 }
1002
1003 static  __checkReturn           efx_rc_t
1004 siena_filter_push_entry(
1005         __inout                 efx_nic_t *enp,
1006         __in                    siena_filter_type_t type,
1007         __in                    int index,
1008         __in                    efx_oword_t *eop)
1009 {
1010         efx_rc_t rc;
1011
1012         switch (type) {
1013         case EFX_SIENA_FILTER_RX_TCP_FULL:
1014         case EFX_SIENA_FILTER_RX_TCP_WILD:
1015         case EFX_SIENA_FILTER_RX_UDP_FULL:
1016         case EFX_SIENA_FILTER_RX_UDP_WILD:
1017                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1018                     eop, B_TRUE);
1019                 break;
1020
1021         case EFX_SIENA_FILTER_RX_MAC_FULL:
1022         case EFX_SIENA_FILTER_RX_MAC_WILD:
1023                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1024                     eop, B_TRUE);
1025                 break;
1026
1027         case EFX_SIENA_FILTER_TX_TCP_FULL:
1028         case EFX_SIENA_FILTER_TX_TCP_WILD:
1029         case EFX_SIENA_FILTER_TX_UDP_FULL:
1030         case EFX_SIENA_FILTER_TX_UDP_WILD:
1031                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1032                     eop, B_TRUE);
1033                 break;
1034
1035         case EFX_SIENA_FILTER_TX_MAC_FULL:
1036         case EFX_SIENA_FILTER_TX_MAC_WILD:
1037                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1038                     eop, B_TRUE);
1039                 break;
1040
1041         default:
1042                 EFSYS_ASSERT(B_FALSE);
1043                 rc = ENOTSUP;
1044                 goto fail1;
1045         }
1046         return (0);
1047
1048 fail1:
1049         return (rc);
1050 }
1051
1052
1053 static  __checkReturn   boolean_t
1054 siena_filter_equal(
1055         __in            const siena_filter_spec_t *left,
1056         __in            const siena_filter_spec_t *right)
1057 {
1058         siena_filter_tbl_id_t tbl_id;
1059
1060         tbl_id = siena_filter_tbl_id(left->sfs_type);
1061
1062
1063         if (left->sfs_type != right->sfs_type)
1064                 return (B_FALSE);
1065
1066         if (memcmp(left->sfs_dword, right->sfs_dword,
1067                 sizeof (left->sfs_dword)))
1068                 return (B_FALSE);
1069
1070         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1071                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1072             left->sfs_dmaq_id != right->sfs_dmaq_id)
1073                 return (B_FALSE);
1074
1075         return (B_TRUE);
1076 }
1077
1078 static  __checkReturn   efx_rc_t
1079 siena_filter_search(
1080         __in            siena_filter_tbl_t *sftp,
1081         __in            siena_filter_spec_t *spec,
1082         __in            uint32_t key,
1083         __in            boolean_t for_insert,
1084         __out           int *filter_index,
1085         __out           unsigned int *depth_required)
1086 {
1087         unsigned int hash, incr, filter_idx, depth;
1088
1089         hash = siena_filter_tbl_hash(key);
1090         incr = siena_filter_tbl_increment(key);
1091
1092         filter_idx = hash & (sftp->sft_size - 1);
1093         depth = 1;
1094
1095         for (;;) {
1096                 /*
1097                  * Return success if entry is used and matches this spec
1098                  * or entry is unused and we are trying to insert.
1099                  */
1100                 if (siena_filter_test_used(sftp, filter_idx) ?
1101                     siena_filter_equal(spec,
1102                     &sftp->sft_spec[filter_idx]) :
1103                     for_insert) {
1104                         *filter_index = filter_idx;
1105                         *depth_required = depth;
1106                         return (0);
1107                 }
1108
1109                 /* Return failure if we reached the maximum search depth */
1110                 if (depth == FILTER_CTL_SRCH_MAX)
1111                         return (for_insert ? EBUSY : ENOENT);
1112
1113                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1114                 ++depth;
1115         }
1116 }
1117
1118 static                  void
1119 siena_filter_clear_entry(
1120         __in            efx_nic_t *enp,
1121         __in            siena_filter_tbl_t *sftp,
1122         __in            int index)
1123 {
1124         efx_oword_t filter;
1125
1126         if (siena_filter_test_used(sftp, index)) {
1127                 siena_filter_clear_used(sftp, index);
1128
1129                 EFX_ZERO_OWORD(filter);
1130                 siena_filter_push_entry(enp,
1131                     sftp->sft_spec[index].sfs_type,
1132                     index, &filter);
1133
1134                 memset(&sftp->sft_spec[index],
1135                     0, sizeof (sftp->sft_spec[0]));
1136         }
1137 }
1138
1139                         void
1140 siena_filter_tbl_clear(
1141         __in            efx_nic_t *enp,
1142         __in            siena_filter_tbl_id_t tbl_id)
1143 {
1144         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1145         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1146         int index;
1147         efsys_lock_state_t state;
1148
1149         EFSYS_LOCK(enp->en_eslp, state);
1150
1151         for (index = 0; index < sftp->sft_size; ++index) {
1152                 siena_filter_clear_entry(enp, sftp, index);
1153         }
1154
1155         if (sftp->sft_used == 0)
1156                 siena_filter_reset_search_depth(sfp, tbl_id);
1157
1158         EFSYS_UNLOCK(enp->en_eslp, state);
1159 }
1160
1161 static  __checkReturn   efx_rc_t
1162 siena_filter_init(
1163         __in            efx_nic_t *enp)
1164 {
1165         siena_filter_t *sfp;
1166         siena_filter_tbl_t *sftp;
1167         int tbl_id;
1168         efx_rc_t rc;
1169
1170         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1171
1172         if (!sfp) {
1173                 rc = ENOMEM;
1174                 goto fail1;
1175         }
1176
1177         enp->en_filter.ef_siena_filter = sfp;
1178
1179         switch (enp->en_family) {
1180         case EFX_FAMILY_SIENA:
1181                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1182                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1183
1184                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1185                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1186
1187                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1188                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1189
1190                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1191                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1192                 break;
1193
1194         default:
1195                 rc = ENOTSUP;
1196                 goto fail2;
1197         }
1198
1199         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1200                 unsigned int bitmap_size;
1201
1202                 sftp = &sfp->sf_tbl[tbl_id];
1203                 if (sftp->sft_size == 0)
1204                         continue;
1205
1206                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1207                     sizeof (uint32_t));
1208                 bitmap_size =
1209                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1210
1211                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1212                 if (!sftp->sft_bitmap) {
1213                         rc = ENOMEM;
1214                         goto fail3;
1215                 }
1216
1217                 EFSYS_KMEM_ALLOC(enp->en_esip,
1218                     sftp->sft_size * sizeof (*sftp->sft_spec),
1219                     sftp->sft_spec);
1220                 if (!sftp->sft_spec) {
1221                         rc = ENOMEM;
1222                         goto fail4;
1223                 }
1224                 memset(sftp->sft_spec, 0,
1225                     sftp->sft_size * sizeof (*sftp->sft_spec));
1226         }
1227
1228         return (0);
1229
1230 fail4:
1231         EFSYS_PROBE(fail4);
1232
1233 fail3:
1234         EFSYS_PROBE(fail3);
1235
1236 fail2:
1237         EFSYS_PROBE(fail2);
1238         siena_filter_fini(enp);
1239
1240 fail1:
1241         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1242         return (rc);
1243 }
1244
1245 static                  void
1246 siena_filter_fini(
1247         __in            efx_nic_t *enp)
1248 {
1249         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1250         siena_filter_tbl_id_t tbl_id;
1251
1252         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1253         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1254
1255         if (sfp == NULL)
1256                 return;
1257
1258         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1259                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1260                 unsigned int bitmap_size;
1261
1262                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1263                     sizeof (uint32_t));
1264                 bitmap_size =
1265                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1266
1267                 if (sftp->sft_bitmap != NULL) {
1268                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1269                             sftp->sft_bitmap);
1270                         sftp->sft_bitmap = NULL;
1271                 }
1272
1273                 if (sftp->sft_spec != NULL) {
1274                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1275                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1276                         sftp->sft_spec = NULL;
1277                 }
1278         }
1279
1280         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1281             enp->en_filter.ef_siena_filter);
1282 }
1283
1284 /* Restore filter state after a reset */
1285 static  __checkReturn   efx_rc_t
1286 siena_filter_restore(
1287         __in            efx_nic_t *enp)
1288 {
1289         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1290         siena_filter_tbl_id_t tbl_id;
1291         siena_filter_tbl_t *sftp;
1292         siena_filter_spec_t *spec;
1293         efx_oword_t filter;
1294         int filter_idx;
1295         efsys_lock_state_t state;
1296         uint32_t key;
1297         efx_rc_t rc;
1298
1299         EFSYS_LOCK(enp->en_eslp, state);
1300
1301         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1302                 sftp = &sfp->sf_tbl[tbl_id];
1303                 for (filter_idx = 0;
1304                         filter_idx < sftp->sft_size;
1305                         filter_idx++) {
1306                         if (!siena_filter_test_used(sftp, filter_idx))
1307                                 continue;
1308
1309                         spec = &sftp->sft_spec[filter_idx];
1310                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1311                                 rc = EINVAL;
1312                                 goto fail1;
1313                         }
1314                         if ((rc = siena_filter_push_entry(enp,
1315                                     spec->sfs_type, filter_idx, &filter)) != 0)
1316                                 goto fail2;
1317                 }
1318         }
1319
1320         siena_filter_push_rx_limits(enp);
1321         siena_filter_push_tx_limits(enp);
1322
1323         EFSYS_UNLOCK(enp->en_eslp, state);
1324
1325         return (0);
1326
1327 fail2:
1328         EFSYS_PROBE(fail2);
1329
1330 fail1:
1331         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1332
1333         EFSYS_UNLOCK(enp->en_eslp, state);
1334
1335         return (rc);
1336 }
1337
1338 static   __checkReturn  efx_rc_t
1339 siena_filter_add(
1340         __in            efx_nic_t *enp,
1341         __inout         efx_filter_spec_t *spec,
1342         __in            boolean_t may_replace)
1343 {
1344         efx_rc_t rc;
1345         siena_filter_spec_t sf_spec;
1346         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1347         siena_filter_tbl_id_t tbl_id;
1348         siena_filter_tbl_t *sftp;
1349         siena_filter_spec_t *saved_sf_spec;
1350         efx_oword_t filter;
1351         int filter_idx;
1352         unsigned int depth;
1353         efsys_lock_state_t state;
1354         uint32_t key;
1355
1356
1357         EFSYS_ASSERT3P(spec, !=, NULL);
1358
1359         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1360                 goto fail1;
1361
1362         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1363         sftp = &sfp->sf_tbl[tbl_id];
1364
1365         if (sftp->sft_size == 0) {
1366                 rc = EINVAL;
1367                 goto fail2;
1368         }
1369
1370         key = siena_filter_build(&filter, &sf_spec);
1371
1372         EFSYS_LOCK(enp->en_eslp, state);
1373
1374         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1375             &filter_idx, &depth);
1376         if (rc != 0)
1377                 goto fail3;
1378
1379         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1380         saved_sf_spec = &sftp->sft_spec[filter_idx];
1381
1382         if (siena_filter_test_used(sftp, filter_idx)) {
1383                 if (may_replace == B_FALSE) {
1384                         rc = EEXIST;
1385                         goto fail4;
1386                 }
1387         }
1388         siena_filter_set_used(sftp, filter_idx);
1389         *saved_sf_spec = sf_spec;
1390
1391         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1392                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1393                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1394                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1395                         siena_filter_push_tx_limits(enp);
1396                 else
1397                         siena_filter_push_rx_limits(enp);
1398         }
1399
1400         siena_filter_push_entry(enp, sf_spec.sfs_type,
1401             filter_idx, &filter);
1402
1403         EFSYS_UNLOCK(enp->en_eslp, state);
1404         return (0);
1405
1406 fail4:
1407         EFSYS_PROBE(fail4);
1408
1409 fail3:
1410         EFSYS_UNLOCK(enp->en_eslp, state);
1411         EFSYS_PROBE(fail3);
1412
1413 fail2:
1414         EFSYS_PROBE(fail2);
1415
1416 fail1:
1417         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1418         return (rc);
1419 }
1420
1421 static   __checkReturn  efx_rc_t
1422 siena_filter_delete(
1423         __in            efx_nic_t *enp,
1424         __inout         efx_filter_spec_t *spec)
1425 {
1426         efx_rc_t rc;
1427         siena_filter_spec_t sf_spec;
1428         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1429         siena_filter_tbl_id_t tbl_id;
1430         siena_filter_tbl_t *sftp;
1431         efx_oword_t filter;
1432         int filter_idx;
1433         unsigned int depth;
1434         efsys_lock_state_t state;
1435         uint32_t key;
1436
1437         EFSYS_ASSERT3P(spec, !=, NULL);
1438
1439         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1440                 goto fail1;
1441
1442         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1443         sftp = &sfp->sf_tbl[tbl_id];
1444
1445         key = siena_filter_build(&filter, &sf_spec);
1446
1447         EFSYS_LOCK(enp->en_eslp, state);
1448
1449         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1450             &filter_idx, &depth);
1451         if (rc != 0)
1452                 goto fail2;
1453
1454         siena_filter_clear_entry(enp, sftp, filter_idx);
1455         if (sftp->sft_used == 0)
1456                 siena_filter_reset_search_depth(sfp, tbl_id);
1457
1458         EFSYS_UNLOCK(enp->en_eslp, state);
1459         return (0);
1460
1461 fail2:
1462         EFSYS_UNLOCK(enp->en_eslp, state);
1463         EFSYS_PROBE(fail2);
1464
1465 fail1:
1466         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1467         return (rc);
1468 }
1469
1470 #define SIENA_MAX_SUPPORTED_MATCHES 4
1471
1472 static  __checkReturn   efx_rc_t
1473 siena_filter_supported_filters(
1474         __in                            efx_nic_t *enp,
1475         __out_ecount(buffer_length)     uint32_t *buffer,
1476         __in                            size_t buffer_length,
1477         __out                           size_t *list_lengthp)
1478 {
1479         uint32_t index = 0;
1480         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1481         size_t list_length;
1482         efx_rc_t rc;
1483
1484         rx_matches[index++] =
1485             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1486             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1487             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1488
1489         rx_matches[index++] =
1490             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1491             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1492
1493         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1494                 rx_matches[index++] =
1495                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1496
1497                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1498         }
1499
1500         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1501         list_length = index;
1502
1503         *list_lengthp = list_length;
1504
1505         if (buffer_length < list_length) {
1506                 rc = ENOSPC;
1507                 goto fail1;
1508         }
1509
1510         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1511
1512         return (0);
1513
1514 fail1:
1515         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1516
1517         return (rc);
1518 }
1519
1520 #undef MAX_SUPPORTED
1521
1522 #endif /* EFSYS_OPT_SIENA */
1523
1524 #endif /* EFSYS_OPT_FILTER */