sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include "efx.h"
  37 #include "efx_impl.h"
  38
  39 #if EFSYS_OPT_FILTER
  40
  41 #if EFSYS_OPT_SIENA
  42
  43 static  __checkReturn   efx_rc_t
  44 siena_filter_init(
  45         __in            efx_nic_t *enp);
  46
  47 static                  void
  48 siena_filter_fini(
  49         __in            efx_nic_t *enp);
  50
  51 static  __checkReturn   efx_rc_t
  52 siena_filter_restore(
  53         __in            efx_nic_t *enp);
  54
  55 static  __checkReturn   efx_rc_t
  56 siena_filter_add(
  57         __in            efx_nic_t *enp,
  58         __inout         efx_filter_spec_t *spec,
  59         __in            boolean_t may_replace);
  60
  61 static  __checkReturn   efx_rc_t
  62 siena_filter_delete(
  63         __in            efx_nic_t *enp,
  64         __inout         efx_filter_spec_t *spec);
  65
  66 static  __checkReturn   efx_rc_t
  67 siena_filter_supported_filters(
  68         __in                            efx_nic_t *enp,
  69         __out_ecount(buffer_length)     uint32_t *buffer,
  70         __in                            size_t buffer_length,
  71         __out                           size_t *list_lengthp);
  72
  73 #endif /* EFSYS_OPT_SIENA */
  74
  75 #if EFSYS_OPT_SIENA
  76 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  77         siena_filter_init,              /* efo_init */
  78         siena_filter_fini,              /* efo_fini */
  79         siena_filter_restore,           /* efo_restore */
  80         siena_filter_add,               /* efo_add */
  81         siena_filter_delete,            /* efo_delete */
  82         siena_filter_supported_filters, /* efo_supported_filters */
  83         NULL,                           /* efo_reconfigure */
  84 };
  85 #endif /* EFSYS_OPT_SIENA */
  86
  87 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  88 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  89         ef10_filter_init,               /* efo_init */
  90         ef10_filter_fini,               /* efo_fini */
  91         ef10_filter_restore,            /* efo_restore */
  92         ef10_filter_add,                /* efo_add */
  93         ef10_filter_delete,             /* efo_delete */
  94         ef10_filter_supported_filters,  /* efo_supported_filters */
  95         ef10_filter_reconfigure,        /* efo_reconfigure */
  96 };
  97 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
  98
  99         __checkReturn   efx_rc_t
 100 efx_filter_insert(
 101         __in            efx_nic_t *enp,
 102         __inout         efx_filter_spec_t *spec)
 103 {
 104         const efx_filter_ops_t *efop = enp->en_efop;
 105         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 106         efx_rc_t rc;
 107
 108         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 109         EFSYS_ASSERT3P(spec, !=, NULL);
 110         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 111
 112         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) &&
 113             !encp->enc_filter_action_mark_supported) {
 114                 rc = ENOTSUP;
 115                 goto fail1;
 116         }
 117
 118         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG) &&
 119             !encp->enc_filter_action_flag_supported) {
 120                 rc = ENOTSUP;
 121                 goto fail2;
 122         }
 123
 124         return (efop->efo_add(enp, spec, B_FALSE));
 125
 126 fail2:
 127         EFSYS_PROBE(fail2);
 128 fail1:
 129         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 130
 131         return (rc);
 132 }
 133
 134         __checkReturn   efx_rc_t
 135 efx_filter_remove(
 136         __in            efx_nic_t *enp,
 137         __inout         efx_filter_spec_t *spec)
 138 {
 139         const efx_filter_ops_t *efop = enp->en_efop;
 140
 141         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 142         EFSYS_ASSERT3P(spec, !=, NULL);
 143         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 144
 145         return (efop->efo_delete(enp, spec));
 146 }
 147
 148         __checkReturn   efx_rc_t
 149 efx_filter_restore(
 150         __in            efx_nic_t *enp)
 151 {
 152         efx_rc_t rc;
 153
 154         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 155
 156         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 157                 goto fail1;
 158
 159         return (0);
 160
 161 fail1:
 162         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 163
 164         return (rc);
 165 }
 166
 167         __checkReturn   efx_rc_t
 168 efx_filter_init(
 169         __in            efx_nic_t *enp)
 170 {
 171         const efx_filter_ops_t *efop;
 172         efx_rc_t rc;
 173
 174         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 175         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 176         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 177
 178         switch (enp->en_family) {
 179 #if EFSYS_OPT_SIENA
 180         case EFX_FAMILY_SIENA:
 181                 efop = &__efx_filter_siena_ops;
 182                 break;
 183 #endif /* EFSYS_OPT_SIENA */
 184
 185 #if EFSYS_OPT_HUNTINGTON
 186         case EFX_FAMILY_HUNTINGTON:
 187                 efop = &__efx_filter_ef10_ops;
 188                 break;
 189 #endif /* EFSYS_OPT_HUNTINGTON */
 190
 191 #if EFSYS_OPT_MEDFORD
 192         case EFX_FAMILY_MEDFORD:
 193                 efop = &__efx_filter_ef10_ops;
 194                 break;
 195 #endif /* EFSYS_OPT_MEDFORD */
 196
 197 #if EFSYS_OPT_MEDFORD2
 198         case EFX_FAMILY_MEDFORD2:
 199                 efop = &__efx_filter_ef10_ops;
 200                 break;
 201 #endif /* EFSYS_OPT_MEDFORD2 */
 202
 203         default:
 204                 EFSYS_ASSERT(0);
 205                 rc = ENOTSUP;
 206                 goto fail1;
 207         }
 208
 209         if ((rc = efop->efo_init(enp)) != 0)
 210                 goto fail2;
 211
 212         enp->en_efop = efop;
 213         enp->en_mod_flags |= EFX_MOD_FILTER;
 214         return (0);
 215
 216 fail2:
 217         EFSYS_PROBE(fail2);
 218 fail1:
 219         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 220
 221         enp->en_efop = NULL;
 222         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 223         return (rc);
 224 }
 225
 226                         void
 227 efx_filter_fini(
 228         __in            efx_nic_t *enp)
 229 {
 230         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 231         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 232         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 233
 234         enp->en_efop->efo_fini(enp);
 235
 236         enp->en_efop = NULL;
 237         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 238 }
 239
 240 /*
 241  * Query the possible combinations of match flags which can be filtered on.
 242  * These are returned as a list, of which each 32 bit element is a bitmask
 243  * formed of EFX_FILTER_MATCH flags.
 244  *
 245  * The combinations are ordered in priority from highest to lowest.
 246  *
 247  * If the provided buffer is too short to hold the list, the call with fail with
 248  * ENOSPC and *list_lengthp will be set to the buffer length required.
 249  */
 250         __checkReturn   efx_rc_t
 251 efx_filter_supported_filters(
 252         __in                            efx_nic_t *enp,
 253         __out_ecount(buffer_length)     uint32_t *buffer,
 254         __in                            size_t buffer_length,
 255         __out                           size_t *list_lengthp)
 256 {
 257         efx_rc_t rc;
 258
 259         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 260         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 261         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 262         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 263
 264         if (buffer == NULL) {
 265                 rc = EINVAL;
 266                 goto fail1;
 267         }
 268
 269         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 270                                                     list_lengthp);
 271         if (rc != 0)
 272                 goto fail2;
 273
 274         return (0);
 275
 276 fail2:
 277         EFSYS_PROBE(fail2);
 278 fail1:
 279         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 280
 281         return (rc);
 282 }
 283
 284         __checkReturn   efx_rc_t
 285 efx_filter_reconfigure(
 286         __in                            efx_nic_t *enp,
 287         __in_ecount(6)                  uint8_t const *mac_addr,
 288         __in                            boolean_t all_unicst,
 289         __in                            boolean_t mulcst,
 290         __in                            boolean_t all_mulcst,
 291         __in                            boolean_t brdcst,
 292         __in_ecount(6*count)            uint8_t const *addrs,
 293         __in                            uint32_t count)
 294 {
 295         efx_rc_t rc;
 296
 297         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 298         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 299         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 300
 301         if (enp->en_efop->efo_reconfigure != NULL) {
 302                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 303                                                         all_unicst, mulcst,
 304                                                         all_mulcst, brdcst,
 305                                                         addrs, count)) != 0)
 306                         goto fail1;
 307         }
 308
 309         return (0);
 310
 311 fail1:
 312         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 313
 314         return (rc);
 315 }
 316
 317                 void
 318 efx_filter_spec_init_rx(
 319         __out           efx_filter_spec_t *spec,
 320         __in            efx_filter_priority_t priority,
 321         __in            efx_filter_flags_t flags,
 322         __in            efx_rxq_t *erp)
 323 {
 324         EFSYS_ASSERT3P(spec, !=, NULL);
 325         EFSYS_ASSERT3P(erp, !=, NULL);
 326         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 327                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 328
 329         memset(spec, 0, sizeof (*spec));
 330         spec->efs_priority = priority;
 331         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 332         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 333         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 334 }
 335
 336                 void
 337 efx_filter_spec_init_tx(
 338         __out           efx_filter_spec_t *spec,
 339         __in            efx_txq_t *etp)
 340 {
 341         EFSYS_ASSERT3P(spec, !=, NULL);
 342         EFSYS_ASSERT3P(etp, !=, NULL);
 343
 344         memset(spec, 0, sizeof (*spec));
 345         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 346         spec->efs_flags = EFX_FILTER_FLAG_TX;
 347         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 348 }
 349
 350 /*
 351  *  Specify IPv4 host, transport protocol and port in a filter specification
 352  */
 353 __checkReturn           efx_rc_t
 354 efx_filter_spec_set_ipv4_local(
 355         __inout         efx_filter_spec_t *spec,
 356         __in            uint8_t proto,
 357         __in            uint32_t host,
 358         __in            uint16_t port)
 359 {
 360         EFSYS_ASSERT3P(spec, !=, NULL);
 361
 362         spec->efs_match_flags |=
 363                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 364                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 365         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 366         spec->efs_ip_proto = proto;
 367         spec->efs_loc_host.eo_u32[0] = host;
 368         spec->efs_loc_port = port;
 369         return (0);
 370 }
 371
 372 /*
 373  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 374  */
 375 __checkReturn           efx_rc_t
 376 efx_filter_spec_set_ipv4_full(
 377         __inout         efx_filter_spec_t *spec,
 378         __in            uint8_t proto,
 379         __in            uint32_t lhost,
 380         __in            uint16_t lport,
 381         __in            uint32_t rhost,
 382         __in            uint16_t rport)
 383 {
 384         EFSYS_ASSERT3P(spec, !=, NULL);
 385
 386         spec->efs_match_flags |=
 387                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 388                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 389                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 390         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 391         spec->efs_ip_proto = proto;
 392         spec->efs_loc_host.eo_u32[0] = lhost;
 393         spec->efs_loc_port = lport;
 394         spec->efs_rem_host.eo_u32[0] = rhost;
 395         spec->efs_rem_port = rport;
 396         return (0);
 397 }
 398
 399 /*
 400  * Specify local Ethernet address and/or VID in filter specification
 401  */
 402 __checkReturn           efx_rc_t
 403 efx_filter_spec_set_eth_local(
 404         __inout         efx_filter_spec_t *spec,
 405         __in            uint16_t vid,
 406         __in            const uint8_t *addr)
 407 {
 408         EFSYS_ASSERT3P(spec, !=, NULL);
 409         EFSYS_ASSERT3P(addr, !=, NULL);
 410
 411         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 412                 return (EINVAL);
 413
 414         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 415                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 416                 spec->efs_outer_vid = vid;
 417         }
 418         if (addr != NULL) {
 419                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 420                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 421         }
 422         return (0);
 423 }
 424
 425                         void
 426 efx_filter_spec_set_ether_type(
 427         __inout         efx_filter_spec_t *spec,
 428         __in            uint16_t ether_type)
 429 {
 430         EFSYS_ASSERT3P(spec, !=, NULL);
 431
 432         spec->efs_ether_type = ether_type;
 433         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 434 }
 435
 436 /*
 437  * Specify matching otherwise-unmatched unicast in a filter specification
 438  */
 439 __checkReturn           efx_rc_t
 440 efx_filter_spec_set_uc_def(
 441         __inout         efx_filter_spec_t *spec)
 442 {
 443         EFSYS_ASSERT3P(spec, !=, NULL);
 444
 445         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 446         return (0);
 447 }
 448
 449 /*
 450  * Specify matching otherwise-unmatched multicast in a filter specification
 451  */
 452 __checkReturn           efx_rc_t
 453 efx_filter_spec_set_mc_def(
 454         __inout         efx_filter_spec_t *spec)
 455 {
 456         EFSYS_ASSERT3P(spec, !=, NULL);
 457
 458         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 459         return (0);
 460 }
 461
 462 __checkReturn           efx_rc_t
 463 efx_filter_spec_set_encap_type(
 464         __inout         efx_filter_spec_t *spec,
 465         __in            efx_tunnel_protocol_t encap_type,
 466         __in            efx_filter_inner_frame_match_t inner_frame_match)
 467 {
 468         uint32_t match_flags = EFX_FILTER_MATCH_ENCAP_TYPE;
 469         uint8_t ip_proto;
 470         efx_rc_t rc;
 471
 472         EFSYS_ASSERT3P(spec, !=, NULL);
 473
 474         switch (encap_type) {
 475         case EFX_TUNNEL_PROTOCOL_VXLAN:
 476         case EFX_TUNNEL_PROTOCOL_GENEVE:
 477                 ip_proto = EFX_IPPROTO_UDP;
 478                 break;
 479         case EFX_TUNNEL_PROTOCOL_NVGRE:
 480                 ip_proto = EFX_IPPROTO_GRE;
 481                 break;
 482         default:
 483                 EFSYS_ASSERT(0);
 484                 rc = EINVAL;
 485                 goto fail1;
 486         }
 487
 488         switch (inner_frame_match) {
 489         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 490                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 491                 break;
 492         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 493                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 494                 break;
 495         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 496                 /* This is for when specific inner frames are to be matched. */
 497                 break;
 498         default:
 499                 EFSYS_ASSERT(0);
 500                 rc = EINVAL;
 501                 goto fail2;
 502         }
 503
 504         spec->efs_encap_type = encap_type;
 505         spec->efs_ip_proto = ip_proto;
 506         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 507
 508         return (0);
 509
 510 fail2:
 511         EFSYS_PROBE(fail2);
 512 fail1:
 513         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 514
 515         return (rc);
 516 }
 517
 518 /*
 519  * Specify inner and outer Ethernet address and VNI or VSID in tunnel filter
 520  * specification.
 521  */
 522 static  __checkReturn   efx_rc_t
 523 efx_filter_spec_set_tunnel(
 524         __inout efx_filter_spec_t *spec,
 525         __in            efx_tunnel_protocol_t encap_type,
 526         __in            const uint8_t *vni_or_vsid,
 527         __in            const uint8_t *inner_addr,
 528         __in            const uint8_t *outer_addr)
 529 {
 530         efx_rc_t rc;
 531
 532         EFSYS_ASSERT3P(spec, !=, NULL);
 533         EFSYS_ASSERT3P(vni_or_vsid, !=, NULL);
 534         EFSYS_ASSERT3P(inner_addr, !=, NULL);
 535         EFSYS_ASSERT3P(outer_addr, !=, NULL);
 536
 537         switch (encap_type) {
 538         case EFX_TUNNEL_PROTOCOL_VXLAN:
 539         case EFX_TUNNEL_PROTOCOL_GENEVE:
 540         case EFX_TUNNEL_PROTOCOL_NVGRE:
 541                 break;
 542         default:
 543                 rc = EINVAL;
 544                 goto fail1;
 545         }
 546
 547         if ((inner_addr == NULL) && (outer_addr == NULL)) {
 548                 rc = EINVAL;
 549                 goto fail2;
 550         }
 551
 552         if (vni_or_vsid != NULL) {
 553                 spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 554                 memcpy(spec->efs_vni_or_vsid, vni_or_vsid, EFX_VNI_OR_VSID_LEN);
 555         }
 556         if (outer_addr != NULL) {
 557                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 558                 memcpy(spec->efs_loc_mac, outer_addr, EFX_MAC_ADDR_LEN);
 559         }
 560         if (inner_addr != NULL) {
 561                 spec->efs_match_flags |= EFX_FILTER_MATCH_IFRM_LOC_MAC;
 562                 memcpy(spec->efs_ifrm_loc_mac, inner_addr, EFX_MAC_ADDR_LEN);
 563         }
 564
 565         spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 566         spec->efs_encap_type = encap_type;
 567
 568         return (0);
 569
 570 fail2:
 571         EFSYS_PROBE(fail2);
 572 fail1:
 573         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 574
 575         return (rc);
 576 }
 577
 578 /*
 579  * Specify inner and outer Ethernet address and VNI in VXLAN filter
 580  * specification.
 581  */
 582 __checkReturn           efx_rc_t
 583 efx_filter_spec_set_vxlan(
 584         __inout         efx_filter_spec_t *spec,
 585         __in            const uint8_t *vni,
 586         __in            const uint8_t *inner_addr,
 587         __in            const uint8_t *outer_addr)
 588 {
 589         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_VXLAN,
 590             vni, inner_addr, outer_addr);
 591 }
 592
 593 /*
 594  * Specify inner and outer Ethernet address and VNI in Geneve filter
 595  * specification.
 596  */
 597 __checkReturn           efx_rc_t
 598 efx_filter_spec_set_geneve(
 599         __inout         efx_filter_spec_t *spec,
 600         __in            const uint8_t *vni,
 601         __in            const uint8_t *inner_addr,
 602         __in            const uint8_t *outer_addr)
 603 {
 604         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_GENEVE,
 605             vni, inner_addr, outer_addr);
 606 }
 607
 608 /*
 609  * Specify inner and outer Ethernet address and vsid in NVGRE filter
 610  * specification.
 611  */
 612 __checkReturn           efx_rc_t
 613 efx_filter_spec_set_nvgre(
 614         __inout         efx_filter_spec_t *spec,
 615         __in            const uint8_t *vsid,
 616         __in            const uint8_t *inner_addr,
 617         __in            const uint8_t *outer_addr)
 618 {
 619         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_NVGRE,
 620             vsid, inner_addr, outer_addr);
 621 }
 622
 623 #if EFSYS_OPT_RX_SCALE
 624         __checkReturn   efx_rc_t
 625 efx_filter_spec_set_rss_context(
 626         __inout         efx_filter_spec_t *spec,
 627         __in            uint32_t rss_context)
 628 {
 629         efx_rc_t rc;
 630
 631         EFSYS_ASSERT3P(spec, !=, NULL);
 632
 633         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 634         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 635                 rc = EINVAL;
 636                 goto fail1;
 637         }
 638
 639         spec->efs_rss_context = rss_context;
 640
 641         return (0);
 642
 643 fail1:
 644         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 645
 646         return (rc);
 647 }
 648 #endif
 649
 650 #if EFSYS_OPT_SIENA
 651
 652 /*
 653  * "Fudge factors" - difference between programmed value and actual depth.
 654  * Due to pipelined implementation we need to program H/W with a value that
 655  * is larger than the hop limit we want.
 656  */
 657 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 658 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 659
 660 /*
 661  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 662  * We also need to avoid infinite loops in efx_filter_search() when the
 663  * table is full.
 664  */
 665 #define FILTER_CTL_SRCH_MAX 200
 666
 667 static  __checkReturn   efx_rc_t
 668 siena_filter_spec_from_gen_spec(
 669         __out           siena_filter_spec_t *sf_spec,
 670         __in            efx_filter_spec_t *gen_spec)
 671 {
 672         efx_rc_t rc;
 673         boolean_t is_full = B_FALSE;
 674
 675         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 676                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 677         else
 678                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 679
 680         /* Siena only has one RSS context */
 681         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 682             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 683                 rc = EINVAL;
 684                 goto fail1;
 685         }
 686
 687         sf_spec->sfs_flags = gen_spec->efs_flags;
 688         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 689
 690         switch (gen_spec->efs_match_flags) {
 691         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 692             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 693             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 694                 is_full = B_TRUE;
 695                 /* Fall through */
 696         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 697             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 698                 uint32_t rhost, host1, host2;
 699                 uint16_t rport, port1, port2;
 700
 701                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 702                         rc = ENOTSUP;
 703                         goto fail2;
 704                 }
 705                 if (gen_spec->efs_loc_port == 0 ||
 706                     (is_full && gen_spec->efs_rem_port == 0)) {
 707                         rc = EINVAL;
 708                         goto fail3;
 709                 }
 710                 switch (gen_spec->efs_ip_proto) {
 711                 case EFX_IPPROTO_TCP:
 712                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 713                                 sf_spec->sfs_type = (is_full ?
 714                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 715                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 716                         } else {
 717                                 sf_spec->sfs_type = (is_full ?
 718                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 719                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 720                         }
 721                         break;
 722                 case EFX_IPPROTO_UDP:
 723                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 724                                 sf_spec->sfs_type = (is_full ?
 725                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 726                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 727                         } else {
 728                                 sf_spec->sfs_type = (is_full ?
 729                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 730                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 731                         }
 732                         break;
 733                 default:
 734                         rc = ENOTSUP;
 735                         goto fail4;
 736                 }
 737                 /*
 738                  * The filter is constructed in terms of source and destination,
 739                  * with the odd wrinkle that the ports are swapped in a UDP
 740                  * wildcard filter. We need to convert from local and remote
 741                  * addresses (zero for a wildcard).
 742                  */
 743                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 744                 rport = is_full ? gen_spec->efs_rem_port : 0;
 745                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 746                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 747                         host2 = rhost;
 748                 } else {
 749                         host1 = rhost;
 750                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 751                 }
 752                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 753                         if (sf_spec->sfs_type ==
 754                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 755                                 port1 = rport;
 756                                 port2 = gen_spec->efs_loc_port;
 757                         } else {
 758                                 port1 = gen_spec->efs_loc_port;
 759                                 port2 = rport;
 760                         }
 761                 } else {
 762                         if (sf_spec->sfs_type ==
 763                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 764                                 port1 = gen_spec->efs_loc_port;
 765                                 port2 = rport;
 766                         } else {
 767                                 port1 = rport;
 768                                 port2 = gen_spec->efs_loc_port;
 769                         }
 770                 }
 771                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 772                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 773                 sf_spec->sfs_dword[2] = host2;
 774                 break;
 775         }
 776
 777         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 778                 is_full = B_TRUE;
 779                 /* Fall through */
 780         case EFX_FILTER_MATCH_LOC_MAC:
 781                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 782                         sf_spec->sfs_type = (is_full ?
 783                             EFX_SIENA_FILTER_TX_MAC_FULL :
 784                             EFX_SIENA_FILTER_TX_MAC_WILD);
 785                 } else {
 786                         sf_spec->sfs_type = (is_full ?
 787                             EFX_SIENA_FILTER_RX_MAC_FULL :
 788                             EFX_SIENA_FILTER_RX_MAC_WILD);
 789                 }
 790                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 791                 sf_spec->sfs_dword[1] =
 792                     gen_spec->efs_loc_mac[2] << 24 |
 793                     gen_spec->efs_loc_mac[3] << 16 |
 794                     gen_spec->efs_loc_mac[4] <<  8 |
 795                     gen_spec->efs_loc_mac[5];
 796                 sf_spec->sfs_dword[2] =
 797                     gen_spec->efs_loc_mac[0] << 8 |
 798                     gen_spec->efs_loc_mac[1];
 799                 break;
 800
 801         default:
 802                 EFSYS_ASSERT(B_FALSE);
 803                 rc = ENOTSUP;
 804                 goto fail5;
 805         }
 806
 807         return (0);
 808
 809 fail5:
 810         EFSYS_PROBE(fail5);
 811 fail4:
 812         EFSYS_PROBE(fail4);
 813 fail3:
 814         EFSYS_PROBE(fail3);
 815 fail2:
 816         EFSYS_PROBE(fail2);
 817 fail1:
 818         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 819
 820         return (rc);
 821 }
 822
 823 /*
 824  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 825  * key derived from the n-tuple.
 826  */
 827 static                  uint16_t
 828 siena_filter_tbl_hash(
 829         __in            uint32_t key)
 830 {
 831         uint16_t tmp;
 832
 833         /* First 16 rounds */
 834         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 835         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 836         tmp = tmp ^ tmp >> 9;
 837
 838         /* Last 16 rounds */
 839         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 840         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 841         tmp = tmp ^ tmp >> 9;
 842
 843         return (tmp);
 844 }
 845
 846 /*
 847  * To allow for hash collisions, filter search continues at these
 848  * increments from the first possible entry selected by the hash.
 849  */
 850 static                  uint16_t
 851 siena_filter_tbl_increment(
 852         __in            uint32_t key)
 853 {
 854         return ((uint16_t)(key * 2 - 1));
 855 }
 856
 857 static  __checkReturn   boolean_t
 858 siena_filter_test_used(
 859         __in            siena_filter_tbl_t *sftp,
 860         __in            unsigned int index)
 861 {
 862         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 863         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 864 }
 865
 866 static                  void
 867 siena_filter_set_used(
 868         __in            siena_filter_tbl_t *sftp,
 869         __in            unsigned int index)
 870 {
 871         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 872         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 873         ++sftp->sft_used;
 874 }
 875
 876 static                  void
 877 siena_filter_clear_used(
 878         __in            siena_filter_tbl_t *sftp,
 879         __in            unsigned int index)
 880 {
 881         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 882         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 883
 884         --sftp->sft_used;
 885         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 886 }
 887
 888 static                  siena_filter_tbl_id_t
 889 siena_filter_tbl_id(
 890         __in            siena_filter_type_t type)
 891 {
 892         siena_filter_tbl_id_t tbl_id;
 893
 894         switch (type) {
 895         case EFX_SIENA_FILTER_RX_TCP_FULL:
 896         case EFX_SIENA_FILTER_RX_TCP_WILD:
 897         case EFX_SIENA_FILTER_RX_UDP_FULL:
 898         case EFX_SIENA_FILTER_RX_UDP_WILD:
 899                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 900                 break;
 901
 902         case EFX_SIENA_FILTER_RX_MAC_FULL:
 903         case EFX_SIENA_FILTER_RX_MAC_WILD:
 904                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 905                 break;
 906
 907         case EFX_SIENA_FILTER_TX_TCP_FULL:
 908         case EFX_SIENA_FILTER_TX_TCP_WILD:
 909         case EFX_SIENA_FILTER_TX_UDP_FULL:
 910         case EFX_SIENA_FILTER_TX_UDP_WILD:
 911                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 912                 break;
 913
 914         case EFX_SIENA_FILTER_TX_MAC_FULL:
 915         case EFX_SIENA_FILTER_TX_MAC_WILD:
 916                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 917                 break;
 918
 919         default:
 920                 EFSYS_ASSERT(B_FALSE);
 921                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 922                 break;
 923         }
 924         return (tbl_id);
 925 }
 926
 927 static                  void
 928 siena_filter_reset_search_depth(
 929         __inout         siena_filter_t *sfp,
 930         __in            siena_filter_tbl_id_t tbl_id)
 931 {
 932         switch (tbl_id) {
 933         case EFX_SIENA_FILTER_TBL_RX_IP:
 934                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 935                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 936                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 937                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 938                 break;
 939
 940         case EFX_SIENA_FILTER_TBL_RX_MAC:
 941                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 942                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 943                 break;
 944
 945         case EFX_SIENA_FILTER_TBL_TX_IP:
 946                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 947                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 948                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 949                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 950                 break;
 951
 952         case EFX_SIENA_FILTER_TBL_TX_MAC:
 953                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 954                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 955                 break;
 956
 957         default:
 958                 EFSYS_ASSERT(B_FALSE);
 959                 break;
 960         }
 961 }
 962
 963 static                  void
 964 siena_filter_push_rx_limits(
 965         __in            efx_nic_t *enp)
 966 {
 967         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 968         efx_oword_t oword;
 969
 970         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 971
 972         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 973             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 974             FILTER_CTL_SRCH_FUDGE_FULL);
 975         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 976             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 977             FILTER_CTL_SRCH_FUDGE_WILD);
 978         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 979             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 980             FILTER_CTL_SRCH_FUDGE_FULL);
 981         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 982             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 983             FILTER_CTL_SRCH_FUDGE_WILD);
 984
 985         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 986                 EFX_SET_OWORD_FIELD(oword,
 987                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 988                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 989                     FILTER_CTL_SRCH_FUDGE_FULL);
 990                 EFX_SET_OWORD_FIELD(oword,
 991                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 992                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 993                     FILTER_CTL_SRCH_FUDGE_WILD);
 994         }
 995
 996         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 997 }
 998
 999 static                  void
1000 siena_filter_push_tx_limits(
1001         __in            efx_nic_t *enp)
1002 {
1003         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1004         efx_oword_t oword;
1005
1006         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
1007
1008         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
1009                 EFX_SET_OWORD_FIELD(oword,
1010                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
1011                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
1012                     FILTER_CTL_SRCH_FUDGE_FULL);
1013                 EFX_SET_OWORD_FIELD(oword,
1014                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
1015                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
1016                     FILTER_CTL_SRCH_FUDGE_WILD);
1017                 EFX_SET_OWORD_FIELD(oword,
1018                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
1019                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
1020                     FILTER_CTL_SRCH_FUDGE_FULL);
1021                 EFX_SET_OWORD_FIELD(oword,
1022                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
1023                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
1024                     FILTER_CTL_SRCH_FUDGE_WILD);
1025         }
1026
1027         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
1028                 EFX_SET_OWORD_FIELD(
1029                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
1030                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
1031                         FILTER_CTL_SRCH_FUDGE_FULL);
1032                 EFX_SET_OWORD_FIELD(
1033                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
1034                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
1035                         FILTER_CTL_SRCH_FUDGE_WILD);
1036         }
1037
1038         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
1039 }
1040
1041 /* Build a filter entry and return its n-tuple key. */
1042 static  __checkReturn   uint32_t
1043 siena_filter_build(
1044         __out           efx_oword_t *filter,
1045         __in            siena_filter_spec_t *spec)
1046 {
1047         uint32_t dword3;
1048         uint32_t key;
1049         uint8_t  type  = spec->sfs_type;
1050         uint32_t flags = spec->sfs_flags;
1051
1052         switch (siena_filter_tbl_id(type)) {
1053         case EFX_SIENA_FILTER_TBL_RX_IP: {
1054                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
1055                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
1056                 EFX_POPULATE_OWORD_7(*filter,
1057                     FRF_BZ_RSS_EN,
1058                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1059                     FRF_BZ_SCATTER_EN,
1060                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1061                     FRF_AZ_TCP_UDP, is_udp,
1062                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
1063                     EFX_DWORD_2, spec->sfs_dword[2],
1064                     EFX_DWORD_1, spec->sfs_dword[1],
1065                     EFX_DWORD_0, spec->sfs_dword[0]);
1066                 dword3 = is_udp;
1067                 break;
1068         }
1069
1070         case EFX_SIENA_FILTER_TBL_RX_MAC: {
1071                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
1072                 EFX_POPULATE_OWORD_7(*filter,
1073                     FRF_CZ_RMFT_RSS_EN,
1074                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1075                     FRF_CZ_RMFT_SCATTER_EN,
1076                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1077                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
1078                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
1079                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
1080                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
1081                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
1082                 dword3 = is_wild;
1083                 break;
1084         }
1085
1086         case EFX_SIENA_FILTER_TBL_TX_IP: {
1087                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
1088                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
1089                 EFX_POPULATE_OWORD_5(*filter,
1090                     FRF_CZ_TIFT_TCP_UDP, is_udp,
1091                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
1092                     EFX_DWORD_2, spec->sfs_dword[2],
1093                     EFX_DWORD_1, spec->sfs_dword[1],
1094                     EFX_DWORD_0, spec->sfs_dword[0]);
1095                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
1096                 break;
1097         }
1098
1099         case EFX_SIENA_FILTER_TBL_TX_MAC: {
1100                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
1101                 EFX_POPULATE_OWORD_5(*filter,
1102                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
1103                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
1104                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
1105                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
1106                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
1107                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
1108                 break;
1109         }
1110
1111         default:
1112                 EFSYS_ASSERT(B_FALSE);
1113                 EFX_ZERO_OWORD(*filter);
1114                 return (0);
1115         }
1116
1117         key =
1118             spec->sfs_dword[0] ^
1119             spec->sfs_dword[1] ^
1120             spec->sfs_dword[2] ^
1121             dword3;
1122
1123         return (key);
1124 }
1125
1126 static  __checkReturn           efx_rc_t
1127 siena_filter_push_entry(
1128         __inout                 efx_nic_t *enp,
1129         __in                    siena_filter_type_t type,
1130         __in                    int index,
1131         __in                    efx_oword_t *eop)
1132 {
1133         efx_rc_t rc;
1134
1135         switch (type) {
1136         case EFX_SIENA_FILTER_RX_TCP_FULL:
1137         case EFX_SIENA_FILTER_RX_TCP_WILD:
1138         case EFX_SIENA_FILTER_RX_UDP_FULL:
1139         case EFX_SIENA_FILTER_RX_UDP_WILD:
1140                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1141                     eop, B_TRUE);
1142                 break;
1143
1144         case EFX_SIENA_FILTER_RX_MAC_FULL:
1145         case EFX_SIENA_FILTER_RX_MAC_WILD:
1146                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1147                     eop, B_TRUE);
1148                 break;
1149
1150         case EFX_SIENA_FILTER_TX_TCP_FULL:
1151         case EFX_SIENA_FILTER_TX_TCP_WILD:
1152         case EFX_SIENA_FILTER_TX_UDP_FULL:
1153         case EFX_SIENA_FILTER_TX_UDP_WILD:
1154                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1155                     eop, B_TRUE);
1156                 break;
1157
1158         case EFX_SIENA_FILTER_TX_MAC_FULL:
1159         case EFX_SIENA_FILTER_TX_MAC_WILD:
1160                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1161                     eop, B_TRUE);
1162                 break;
1163
1164         default:
1165                 EFSYS_ASSERT(B_FALSE);
1166                 rc = ENOTSUP;
1167                 goto fail1;
1168         }
1169         return (0);
1170
1171 fail1:
1172         return (rc);
1173 }
1174
1175 static  __checkReturn   boolean_t
1176 siena_filter_equal(
1177         __in            const siena_filter_spec_t *left,
1178         __in            const siena_filter_spec_t *right)
1179 {
1180         siena_filter_tbl_id_t tbl_id;
1181
1182         tbl_id = siena_filter_tbl_id(left->sfs_type);
1183
1184         if (left->sfs_type != right->sfs_type)
1185                 return (B_FALSE);
1186
1187         if (memcmp(left->sfs_dword, right->sfs_dword,
1188                 sizeof (left->sfs_dword)))
1189                 return (B_FALSE);
1190
1191         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1192                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1193             left->sfs_dmaq_id != right->sfs_dmaq_id)
1194                 return (B_FALSE);
1195
1196         return (B_TRUE);
1197 }
1198
1199 static  __checkReturn   efx_rc_t
1200 siena_filter_search(
1201         __in            siena_filter_tbl_t *sftp,
1202         __in            siena_filter_spec_t *spec,
1203         __in            uint32_t key,
1204         __in            boolean_t for_insert,
1205         __out           int *filter_index,
1206         __out           unsigned int *depth_required)
1207 {
1208         unsigned int hash, incr, filter_idx, depth;
1209
1210         hash = siena_filter_tbl_hash(key);
1211         incr = siena_filter_tbl_increment(key);
1212
1213         filter_idx = hash & (sftp->sft_size - 1);
1214         depth = 1;
1215
1216         for (;;) {
1217                 /*
1218                  * Return success if entry is used and matches this spec
1219                  * or entry is unused and we are trying to insert.
1220                  */
1221                 if (siena_filter_test_used(sftp, filter_idx) ?
1222                     siena_filter_equal(spec,
1223                     &sftp->sft_spec[filter_idx]) :
1224                     for_insert) {
1225                         *filter_index = filter_idx;
1226                         *depth_required = depth;
1227                         return (0);
1228                 }
1229
1230                 /* Return failure if we reached the maximum search depth */
1231                 if (depth == FILTER_CTL_SRCH_MAX)
1232                         return (for_insert ? EBUSY : ENOENT);
1233
1234                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1235                 ++depth;
1236         }
1237 }
1238
1239 static                  void
1240 siena_filter_clear_entry(
1241         __in            efx_nic_t *enp,
1242         __in            siena_filter_tbl_t *sftp,
1243         __in            int index)
1244 {
1245         efx_oword_t filter;
1246
1247         if (siena_filter_test_used(sftp, index)) {
1248                 siena_filter_clear_used(sftp, index);
1249
1250                 EFX_ZERO_OWORD(filter);
1251                 siena_filter_push_entry(enp,
1252                     sftp->sft_spec[index].sfs_type,
1253                     index, &filter);
1254
1255                 memset(&sftp->sft_spec[index],
1256                     0, sizeof (sftp->sft_spec[0]));
1257         }
1258 }
1259
1260                         void
1261 siena_filter_tbl_clear(
1262         __in            efx_nic_t *enp,
1263         __in            siena_filter_tbl_id_t tbl_id)
1264 {
1265         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1266         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1267         int index;
1268         efsys_lock_state_t state;
1269
1270         EFSYS_LOCK(enp->en_eslp, state);
1271
1272         for (index = 0; index < sftp->sft_size; ++index) {
1273                 siena_filter_clear_entry(enp, sftp, index);
1274         }
1275
1276         if (sftp->sft_used == 0)
1277                 siena_filter_reset_search_depth(sfp, tbl_id);
1278
1279         EFSYS_UNLOCK(enp->en_eslp, state);
1280 }
1281
1282 static  __checkReturn   efx_rc_t
1283 siena_filter_init(
1284         __in            efx_nic_t *enp)
1285 {
1286         siena_filter_t *sfp;
1287         siena_filter_tbl_t *sftp;
1288         int tbl_id;
1289         efx_rc_t rc;
1290
1291         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1292
1293         if (!sfp) {
1294                 rc = ENOMEM;
1295                 goto fail1;
1296         }
1297
1298         enp->en_filter.ef_siena_filter = sfp;
1299
1300         switch (enp->en_family) {
1301         case EFX_FAMILY_SIENA:
1302                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1303                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1304
1305                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1306                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1307
1308                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1309                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1310
1311                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1312                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1313                 break;
1314
1315         default:
1316                 rc = ENOTSUP;
1317                 goto fail2;
1318         }
1319
1320         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1321                 unsigned int bitmap_size;
1322
1323                 sftp = &sfp->sf_tbl[tbl_id];
1324                 if (sftp->sft_size == 0)
1325                         continue;
1326
1327                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1328                     sizeof (uint32_t));
1329                 bitmap_size =
1330                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1331
1332                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1333                 if (!sftp->sft_bitmap) {
1334                         rc = ENOMEM;
1335                         goto fail3;
1336                 }
1337
1338                 EFSYS_KMEM_ALLOC(enp->en_esip,
1339                     sftp->sft_size * sizeof (*sftp->sft_spec),
1340                     sftp->sft_spec);
1341                 if (!sftp->sft_spec) {
1342                         rc = ENOMEM;
1343                         goto fail4;
1344                 }
1345                 memset(sftp->sft_spec, 0,
1346                     sftp->sft_size * sizeof (*sftp->sft_spec));
1347         }
1348
1349         return (0);
1350
1351 fail4:
1352         EFSYS_PROBE(fail4);
1353
1354 fail3:
1355         EFSYS_PROBE(fail3);
1356
1357 fail2:
1358         EFSYS_PROBE(fail2);
1359         siena_filter_fini(enp);
1360
1361 fail1:
1362         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1363         return (rc);
1364 }
1365
1366 static                  void
1367 siena_filter_fini(
1368         __in            efx_nic_t *enp)
1369 {
1370         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1371         siena_filter_tbl_id_t tbl_id;
1372
1373         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1374         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1375
1376         if (sfp == NULL)
1377                 return;
1378
1379         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1380                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1381                 unsigned int bitmap_size;
1382
1383                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1384                     sizeof (uint32_t));
1385                 bitmap_size =
1386                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1387
1388                 if (sftp->sft_bitmap != NULL) {
1389                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1390                             sftp->sft_bitmap);
1391                         sftp->sft_bitmap = NULL;
1392                 }
1393
1394                 if (sftp->sft_spec != NULL) {
1395                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1396                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1397                         sftp->sft_spec = NULL;
1398                 }
1399         }
1400
1401         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1402             enp->en_filter.ef_siena_filter);
1403 }
1404
1405 /* Restore filter state after a reset */
1406 static  __checkReturn   efx_rc_t
1407 siena_filter_restore(
1408         __in            efx_nic_t *enp)
1409 {
1410         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1411         siena_filter_tbl_id_t tbl_id;
1412         siena_filter_tbl_t *sftp;
1413         siena_filter_spec_t *spec;
1414         efx_oword_t filter;
1415         int filter_idx;
1416         efsys_lock_state_t state;
1417         uint32_t key;
1418         efx_rc_t rc;
1419
1420         EFSYS_LOCK(enp->en_eslp, state);
1421
1422         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1423                 sftp = &sfp->sf_tbl[tbl_id];
1424                 for (filter_idx = 0;
1425                         filter_idx < sftp->sft_size;
1426                         filter_idx++) {
1427                         if (!siena_filter_test_used(sftp, filter_idx))
1428                                 continue;
1429
1430                         spec = &sftp->sft_spec[filter_idx];
1431                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1432                                 rc = EINVAL;
1433                                 goto fail1;
1434                         }
1435                         if ((rc = siena_filter_push_entry(enp,
1436                                     spec->sfs_type, filter_idx, &filter)) != 0)
1437                                 goto fail2;
1438                 }
1439         }
1440
1441         siena_filter_push_rx_limits(enp);
1442         siena_filter_push_tx_limits(enp);
1443
1444         EFSYS_UNLOCK(enp->en_eslp, state);
1445
1446         return (0);
1447
1448 fail2:
1449         EFSYS_PROBE(fail2);
1450
1451 fail1:
1452         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1453
1454         EFSYS_UNLOCK(enp->en_eslp, state);
1455
1456         return (rc);
1457 }
1458
1459 static   __checkReturn  efx_rc_t
1460 siena_filter_add(
1461         __in            efx_nic_t *enp,
1462         __inout         efx_filter_spec_t *spec,
1463         __in            boolean_t may_replace)
1464 {
1465         efx_rc_t rc;
1466         siena_filter_spec_t sf_spec;
1467         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1468         siena_filter_tbl_id_t tbl_id;
1469         siena_filter_tbl_t *sftp;
1470         siena_filter_spec_t *saved_sf_spec;
1471         efx_oword_t filter;
1472         int filter_idx;
1473         unsigned int depth;
1474         efsys_lock_state_t state;
1475         uint32_t key;
1476
1477         EFSYS_ASSERT3P(spec, !=, NULL);
1478
1479         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1480                 goto fail1;
1481
1482         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1483         sftp = &sfp->sf_tbl[tbl_id];
1484
1485         if (sftp->sft_size == 0) {
1486                 rc = EINVAL;
1487                 goto fail2;
1488         }
1489
1490         key = siena_filter_build(&filter, &sf_spec);
1491
1492         EFSYS_LOCK(enp->en_eslp, state);
1493
1494         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1495             &filter_idx, &depth);
1496         if (rc != 0)
1497                 goto fail3;
1498
1499         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1500         saved_sf_spec = &sftp->sft_spec[filter_idx];
1501
1502         if (siena_filter_test_used(sftp, filter_idx)) {
1503                 if (may_replace == B_FALSE) {
1504                         rc = EEXIST;
1505                         goto fail4;
1506                 }
1507         }
1508         siena_filter_set_used(sftp, filter_idx);
1509         *saved_sf_spec = sf_spec;
1510
1511         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1512                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1513                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1514                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1515                         siena_filter_push_tx_limits(enp);
1516                 else
1517                         siena_filter_push_rx_limits(enp);
1518         }
1519
1520         siena_filter_push_entry(enp, sf_spec.sfs_type,
1521             filter_idx, &filter);
1522
1523         EFSYS_UNLOCK(enp->en_eslp, state);
1524         return (0);
1525
1526 fail4:
1527         EFSYS_PROBE(fail4);
1528
1529 fail3:
1530         EFSYS_UNLOCK(enp->en_eslp, state);
1531         EFSYS_PROBE(fail3);
1532
1533 fail2:
1534         EFSYS_PROBE(fail2);
1535
1536 fail1:
1537         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1538         return (rc);
1539 }
1540
1541 static   __checkReturn  efx_rc_t
1542 siena_filter_delete(
1543         __in            efx_nic_t *enp,
1544         __inout         efx_filter_spec_t *spec)
1545 {
1546         efx_rc_t rc;
1547         siena_filter_spec_t sf_spec;
1548         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1549         siena_filter_tbl_id_t tbl_id;
1550         siena_filter_tbl_t *sftp;
1551         efx_oword_t filter;
1552         int filter_idx;
1553         unsigned int depth;
1554         efsys_lock_state_t state;
1555         uint32_t key;
1556
1557         EFSYS_ASSERT3P(spec, !=, NULL);
1558
1559         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1560                 goto fail1;
1561
1562         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1563         sftp = &sfp->sf_tbl[tbl_id];
1564
1565         key = siena_filter_build(&filter, &sf_spec);
1566
1567         EFSYS_LOCK(enp->en_eslp, state);
1568
1569         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1570             &filter_idx, &depth);
1571         if (rc != 0)
1572                 goto fail2;
1573
1574         siena_filter_clear_entry(enp, sftp, filter_idx);
1575         if (sftp->sft_used == 0)
1576                 siena_filter_reset_search_depth(sfp, tbl_id);
1577
1578         EFSYS_UNLOCK(enp->en_eslp, state);
1579         return (0);
1580
1581 fail2:
1582         EFSYS_UNLOCK(enp->en_eslp, state);
1583         EFSYS_PROBE(fail2);
1584
1585 fail1:
1586         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1587         return (rc);
1588 }
1589
1590 #define SIENA_MAX_SUPPORTED_MATCHES 4
1591
1592 static  __checkReturn   efx_rc_t
1593 siena_filter_supported_filters(
1594         __in                            efx_nic_t *enp,
1595         __out_ecount(buffer_length)     uint32_t *buffer,
1596         __in                            size_t buffer_length,
1597         __out                           size_t *list_lengthp)
1598 {
1599         uint32_t index = 0;
1600         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1601         size_t list_length;
1602         efx_rc_t rc;
1603
1604         rx_matches[index++] =
1605             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1606             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1607             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1608
1609         rx_matches[index++] =
1610             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1611             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1612
1613         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1614                 rx_matches[index++] =
1615                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1616
1617                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1618         }
1619
1620         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1621         list_length = index;
1622
1623         *list_lengthp = list_length;
1624
1625         if (buffer_length < list_length) {
1626                 rc = ENOSPC;
1627                 goto fail1;
1628         }
1629
1630         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1631
1632         return (0);
1633
1634 fail1:
1635         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1636
1637         return (rc);
1638 }
1639
1640 #undef MAX_SUPPORTED
1641
1642 #endif /* EFSYS_OPT_SIENA */
1643
1644 #endif /* EFSYS_OPT_FILTER */