sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include "efx.h"
  37 #include "efx_impl.h"
  38
  39
  40 #if EFSYS_OPT_FILTER
  41
  42 #if EFSYS_OPT_SIENA
  43
  44 static  __checkReturn   efx_rc_t
  45 siena_filter_init(
  46         __in            efx_nic_t *enp);
  47
  48 static                  void
  49 siena_filter_fini(
  50         __in            efx_nic_t *enp);
  51
  52 static  __checkReturn   efx_rc_t
  53 siena_filter_restore(
  54         __in            efx_nic_t *enp);
  55
  56 static  __checkReturn   efx_rc_t
  57 siena_filter_add(
  58         __in            efx_nic_t *enp,
  59         __inout         efx_filter_spec_t *spec,
  60         __in            boolean_t may_replace);
  61
  62 static  __checkReturn   efx_rc_t
  63 siena_filter_delete(
  64         __in            efx_nic_t *enp,
  65         __inout         efx_filter_spec_t *spec);
  66
  67 static  __checkReturn   efx_rc_t
  68 siena_filter_supported_filters(
  69         __in                            efx_nic_t *enp,
  70         __out_ecount(buffer_length)     uint32_t *buffer,
  71         __in                            size_t buffer_length,
  72         __out                           size_t *list_lengthp);
  73
  74 #endif /* EFSYS_OPT_SIENA */
  75
  76 #if EFSYS_OPT_SIENA
  77 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  78         siena_filter_init,              /* efo_init */
  79         siena_filter_fini,              /* efo_fini */
  80         siena_filter_restore,           /* efo_restore */
  81         siena_filter_add,               /* efo_add */
  82         siena_filter_delete,            /* efo_delete */
  83         siena_filter_supported_filters, /* efo_supported_filters */
  84         NULL,                           /* efo_reconfigure */
  85 };
  86 #endif /* EFSYS_OPT_SIENA */
  87
  88 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  89 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  90         ef10_filter_init,               /* efo_init */
  91         ef10_filter_fini,               /* efo_fini */
  92         ef10_filter_restore,            /* efo_restore */
  93         ef10_filter_add,                /* efo_add */
  94         ef10_filter_delete,             /* efo_delete */
  95         ef10_filter_supported_filters,  /* efo_supported_filters */
  96         ef10_filter_reconfigure,        /* efo_reconfigure */
  97 };
  98 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
  99
 100         __checkReturn   efx_rc_t
 101 efx_filter_insert(
 102         __in            efx_nic_t *enp,
 103         __inout         efx_filter_spec_t *spec)
 104 {
 105         const efx_filter_ops_t *efop = enp->en_efop;
 106         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 107         efx_rc_t rc;
 108
 109         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 110         EFSYS_ASSERT3P(spec, !=, NULL);
 111         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 112
 113         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) &&
 114             !encp->enc_filter_action_mark_supported) {
 115                 rc = ENOTSUP;
 116                 goto fail1;
 117         }
 118
 119         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG) &&
 120             !encp->enc_filter_action_flag_supported) {
 121                 rc = ENOTSUP;
 122                 goto fail2;
 123         }
 124
 125         return (efop->efo_add(enp, spec, B_FALSE));
 126
 127 fail2:
 128         EFSYS_PROBE(fail2);
 129 fail1:
 130         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 131
 132         return (rc);
 133 }
 134
 135         __checkReturn   efx_rc_t
 136 efx_filter_remove(
 137         __in            efx_nic_t *enp,
 138         __inout         efx_filter_spec_t *spec)
 139 {
 140         const efx_filter_ops_t *efop = enp->en_efop;
 141
 142         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 143         EFSYS_ASSERT3P(spec, !=, NULL);
 144         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 145
 146         return (efop->efo_delete(enp, spec));
 147 }
 148
 149         __checkReturn   efx_rc_t
 150 efx_filter_restore(
 151         __in            efx_nic_t *enp)
 152 {
 153         efx_rc_t rc;
 154
 155         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 156
 157         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 158                 goto fail1;
 159
 160         return (0);
 161
 162 fail1:
 163         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 164
 165         return (rc);
 166 }
 167
 168         __checkReturn   efx_rc_t
 169 efx_filter_init(
 170         __in            efx_nic_t *enp)
 171 {
 172         const efx_filter_ops_t *efop;
 173         efx_rc_t rc;
 174
 175         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 176         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 177         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 178
 179         switch (enp->en_family) {
 180 #if EFSYS_OPT_SIENA
 181         case EFX_FAMILY_SIENA:
 182                 efop = &__efx_filter_siena_ops;
 183                 break;
 184 #endif /* EFSYS_OPT_SIENA */
 185
 186 #if EFSYS_OPT_HUNTINGTON
 187         case EFX_FAMILY_HUNTINGTON:
 188                 efop = &__efx_filter_ef10_ops;
 189                 break;
 190 #endif /* EFSYS_OPT_HUNTINGTON */
 191
 192 #if EFSYS_OPT_MEDFORD
 193         case EFX_FAMILY_MEDFORD:
 194                 efop = &__efx_filter_ef10_ops;
 195                 break;
 196 #endif /* EFSYS_OPT_MEDFORD */
 197
 198 #if EFSYS_OPT_MEDFORD2
 199         case EFX_FAMILY_MEDFORD2:
 200                 efop = &__efx_filter_ef10_ops;
 201                 break;
 202 #endif /* EFSYS_OPT_MEDFORD2 */
 203
 204         default:
 205                 EFSYS_ASSERT(0);
 206                 rc = ENOTSUP;
 207                 goto fail1;
 208         }
 209
 210         if ((rc = efop->efo_init(enp)) != 0)
 211                 goto fail2;
 212
 213         enp->en_efop = efop;
 214         enp->en_mod_flags |= EFX_MOD_FILTER;
 215         return (0);
 216
 217 fail2:
 218         EFSYS_PROBE(fail2);
 219 fail1:
 220         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 221
 222         enp->en_efop = NULL;
 223         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 224         return (rc);
 225 }
 226
 227                         void
 228 efx_filter_fini(
 229         __in            efx_nic_t *enp)
 230 {
 231         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 232         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 233         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 234
 235         enp->en_efop->efo_fini(enp);
 236
 237         enp->en_efop = NULL;
 238         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 239 }
 240
 241 /*
 242  * Query the possible combinations of match flags which can be filtered on.
 243  * These are returned as a list, of which each 32 bit element is a bitmask
 244  * formed of EFX_FILTER_MATCH flags.
 245  *
 246  * The combinations are ordered in priority from highest to lowest.
 247  *
 248  * If the provided buffer is too short to hold the list, the call with fail with
 249  * ENOSPC and *list_lengthp will be set to the buffer length required.
 250  */
 251         __checkReturn   efx_rc_t
 252 efx_filter_supported_filters(
 253         __in                            efx_nic_t *enp,
 254         __out_ecount(buffer_length)     uint32_t *buffer,
 255         __in                            size_t buffer_length,
 256         __out                           size_t *list_lengthp)
 257 {
 258         efx_rc_t rc;
 259
 260         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 261         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 262         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 263         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 264
 265         if (buffer == NULL) {
 266                 rc = EINVAL;
 267                 goto fail1;
 268         }
 269
 270         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 271                                                     list_lengthp);
 272         if (rc != 0)
 273                 goto fail2;
 274
 275         return (0);
 276
 277 fail2:
 278         EFSYS_PROBE(fail2);
 279 fail1:
 280         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 281
 282         return (rc);
 283 }
 284
 285         __checkReturn   efx_rc_t
 286 efx_filter_reconfigure(
 287         __in                            efx_nic_t *enp,
 288         __in_ecount(6)                  uint8_t const *mac_addr,
 289         __in                            boolean_t all_unicst,
 290         __in                            boolean_t mulcst,
 291         __in                            boolean_t all_mulcst,
 292         __in                            boolean_t brdcst,
 293         __in_ecount(6*count)            uint8_t const *addrs,
 294         __in                            uint32_t count)
 295 {
 296         efx_rc_t rc;
 297
 298         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 299         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 300         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 301
 302         if (enp->en_efop->efo_reconfigure != NULL) {
 303                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 304                                                         all_unicst, mulcst,
 305                                                         all_mulcst, brdcst,
 306                                                         addrs, count)) != 0)
 307                         goto fail1;
 308         }
 309
 310         return (0);
 311
 312 fail1:
 313         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 314
 315         return (rc);
 316 }
 317
 318                 void
 319 efx_filter_spec_init_rx(
 320         __out           efx_filter_spec_t *spec,
 321         __in            efx_filter_priority_t priority,
 322         __in            efx_filter_flags_t flags,
 323         __in            efx_rxq_t *erp)
 324 {
 325         EFSYS_ASSERT3P(spec, !=, NULL);
 326         EFSYS_ASSERT3P(erp, !=, NULL);
 327         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 328                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 329
 330         memset(spec, 0, sizeof (*spec));
 331         spec->efs_priority = priority;
 332         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 333         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 334         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 335 }
 336
 337                 void
 338 efx_filter_spec_init_tx(
 339         __out           efx_filter_spec_t *spec,
 340         __in            efx_txq_t *etp)
 341 {
 342         EFSYS_ASSERT3P(spec, !=, NULL);
 343         EFSYS_ASSERT3P(etp, !=, NULL);
 344
 345         memset(spec, 0, sizeof (*spec));
 346         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 347         spec->efs_flags = EFX_FILTER_FLAG_TX;
 348         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 349 }
 350
 351
 352 /*
 353  *  Specify IPv4 host, transport protocol and port in a filter specification
 354  */
 355 __checkReturn           efx_rc_t
 356 efx_filter_spec_set_ipv4_local(
 357         __inout         efx_filter_spec_t *spec,
 358         __in            uint8_t proto,
 359         __in            uint32_t host,
 360         __in            uint16_t port)
 361 {
 362         EFSYS_ASSERT3P(spec, !=, NULL);
 363
 364         spec->efs_match_flags |=
 365                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 366                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 367         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 368         spec->efs_ip_proto = proto;
 369         spec->efs_loc_host.eo_u32[0] = host;
 370         spec->efs_loc_port = port;
 371         return (0);
 372 }
 373
 374 /*
 375  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 376  */
 377 __checkReturn           efx_rc_t
 378 efx_filter_spec_set_ipv4_full(
 379         __inout         efx_filter_spec_t *spec,
 380         __in            uint8_t proto,
 381         __in            uint32_t lhost,
 382         __in            uint16_t lport,
 383         __in            uint32_t rhost,
 384         __in            uint16_t rport)
 385 {
 386         EFSYS_ASSERT3P(spec, !=, NULL);
 387
 388         spec->efs_match_flags |=
 389                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 390                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 391                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 392         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 393         spec->efs_ip_proto = proto;
 394         spec->efs_loc_host.eo_u32[0] = lhost;
 395         spec->efs_loc_port = lport;
 396         spec->efs_rem_host.eo_u32[0] = rhost;
 397         spec->efs_rem_port = rport;
 398         return (0);
 399 }
 400
 401 /*
 402  * Specify local Ethernet address and/or VID in filter specification
 403  */
 404 __checkReturn           efx_rc_t
 405 efx_filter_spec_set_eth_local(
 406         __inout         efx_filter_spec_t *spec,
 407         __in            uint16_t vid,
 408         __in            const uint8_t *addr)
 409 {
 410         EFSYS_ASSERT3P(spec, !=, NULL);
 411         EFSYS_ASSERT3P(addr, !=, NULL);
 412
 413         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 414                 return (EINVAL);
 415
 416         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 417                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 418                 spec->efs_outer_vid = vid;
 419         }
 420         if (addr != NULL) {
 421                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 422                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 423         }
 424         return (0);
 425 }
 426
 427                         void
 428 efx_filter_spec_set_ether_type(
 429         __inout         efx_filter_spec_t *spec,
 430         __in            uint16_t ether_type)
 431 {
 432         EFSYS_ASSERT3P(spec, !=, NULL);
 433
 434         spec->efs_ether_type = ether_type;
 435         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 436 }
 437
 438 /*
 439  * Specify matching otherwise-unmatched unicast in a filter specification
 440  */
 441 __checkReturn           efx_rc_t
 442 efx_filter_spec_set_uc_def(
 443         __inout         efx_filter_spec_t *spec)
 444 {
 445         EFSYS_ASSERT3P(spec, !=, NULL);
 446
 447         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 448         return (0);
 449 }
 450
 451 /*
 452  * Specify matching otherwise-unmatched multicast in a filter specification
 453  */
 454 __checkReturn           efx_rc_t
 455 efx_filter_spec_set_mc_def(
 456         __inout         efx_filter_spec_t *spec)
 457 {
 458         EFSYS_ASSERT3P(spec, !=, NULL);
 459
 460         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 461         return (0);
 462 }
 463
 464
 465 __checkReturn           efx_rc_t
 466 efx_filter_spec_set_encap_type(
 467         __inout         efx_filter_spec_t *spec,
 468         __in            efx_tunnel_protocol_t encap_type,
 469         __in            efx_filter_inner_frame_match_t inner_frame_match)
 470 {
 471         uint32_t match_flags = EFX_FILTER_MATCH_ENCAP_TYPE;
 472         uint8_t ip_proto;
 473         efx_rc_t rc;
 474
 475         EFSYS_ASSERT3P(spec, !=, NULL);
 476
 477         switch (encap_type) {
 478         case EFX_TUNNEL_PROTOCOL_VXLAN:
 479         case EFX_TUNNEL_PROTOCOL_GENEVE:
 480                 ip_proto = EFX_IPPROTO_UDP;
 481                 break;
 482         case EFX_TUNNEL_PROTOCOL_NVGRE:
 483                 ip_proto = EFX_IPPROTO_GRE;
 484                 break;
 485         default:
 486                 EFSYS_ASSERT(0);
 487                 rc = EINVAL;
 488                 goto fail1;
 489         }
 490
 491         switch (inner_frame_match) {
 492         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 493                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 494                 break;
 495         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 496                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 497                 break;
 498         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 499                 /* This is for when specific inner frames are to be matched. */
 500                 break;
 501         default:
 502                 EFSYS_ASSERT(0);
 503                 rc = EINVAL;
 504                 goto fail2;
 505         }
 506
 507         spec->efs_encap_type = encap_type;
 508         spec->efs_ip_proto = ip_proto;
 509         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 510
 511         return (0);
 512
 513 fail2:
 514         EFSYS_PROBE(fail2);
 515 fail1:
 516         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 517
 518         return (rc);
 519 }
 520
 521 /*
 522  * Specify inner and outer Ethernet address and VNI or VSID in tunnel filter
 523  * specification.
 524  */
 525 static  __checkReturn   efx_rc_t
 526 efx_filter_spec_set_tunnel(
 527         __inout efx_filter_spec_t *spec,
 528         __in            efx_tunnel_protocol_t encap_type,
 529         __in            const uint8_t *vni_or_vsid,
 530         __in            const uint8_t *inner_addr,
 531         __in            const uint8_t *outer_addr)
 532 {
 533         efx_rc_t rc;
 534
 535         EFSYS_ASSERT3P(spec, !=, NULL);
 536         EFSYS_ASSERT3P(vni_or_vsid, !=, NULL);
 537         EFSYS_ASSERT3P(inner_addr, !=, NULL);
 538         EFSYS_ASSERT3P(outer_addr, !=, NULL);
 539
 540         switch (encap_type) {
 541         case EFX_TUNNEL_PROTOCOL_VXLAN:
 542         case EFX_TUNNEL_PROTOCOL_GENEVE:
 543         case EFX_TUNNEL_PROTOCOL_NVGRE:
 544                 break;
 545         default:
 546                 rc = EINVAL;
 547                 goto fail1;
 548         }
 549
 550         if ((inner_addr == NULL) && (outer_addr == NULL)) {
 551                 rc = EINVAL;
 552                 goto fail2;
 553         }
 554
 555         if (vni_or_vsid != NULL) {
 556                 spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 557                 memcpy(spec->efs_vni_or_vsid, vni_or_vsid, EFX_VNI_OR_VSID_LEN);
 558         }
 559         if (outer_addr != NULL) {
 560                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 561                 memcpy(spec->efs_loc_mac, outer_addr, EFX_MAC_ADDR_LEN);
 562         }
 563         if (inner_addr != NULL) {
 564                 spec->efs_match_flags |= EFX_FILTER_MATCH_IFRM_LOC_MAC;
 565                 memcpy(spec->efs_ifrm_loc_mac, inner_addr, EFX_MAC_ADDR_LEN);
 566         }
 567
 568         spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 569         spec->efs_encap_type = encap_type;
 570
 571         return (0);
 572
 573 fail2:
 574         EFSYS_PROBE(fail2);
 575 fail1:
 576         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 577
 578         return (rc);
 579 }
 580
 581 /*
 582  * Specify inner and outer Ethernet address and VNI in VXLAN filter
 583  * specification.
 584  */
 585 __checkReturn           efx_rc_t
 586 efx_filter_spec_set_vxlan(
 587         __inout         efx_filter_spec_t *spec,
 588         __in            const uint8_t *vni,
 589         __in            const uint8_t *inner_addr,
 590         __in            const uint8_t *outer_addr)
 591 {
 592         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_VXLAN,
 593             vni, inner_addr, outer_addr);
 594 }
 595
 596 /*
 597  * Specify inner and outer Ethernet address and VNI in Geneve filter
 598  * specification.
 599  */
 600 __checkReturn           efx_rc_t
 601 efx_filter_spec_set_geneve(
 602         __inout         efx_filter_spec_t *spec,
 603         __in            const uint8_t *vni,
 604         __in            const uint8_t *inner_addr,
 605         __in            const uint8_t *outer_addr)
 606 {
 607         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_GENEVE,
 608             vni, inner_addr, outer_addr);
 609 }
 610
 611 /*
 612  * Specify inner and outer Ethernet address and vsid in NVGRE filter
 613  * specification.
 614  */
 615 __checkReturn           efx_rc_t
 616 efx_filter_spec_set_nvgre(
 617         __inout         efx_filter_spec_t *spec,
 618         __in            const uint8_t *vsid,
 619         __in            const uint8_t *inner_addr,
 620         __in            const uint8_t *outer_addr)
 621 {
 622         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_NVGRE,
 623             vsid, inner_addr, outer_addr);
 624 }
 625
 626 #if EFSYS_OPT_RX_SCALE
 627         __checkReturn   efx_rc_t
 628 efx_filter_spec_set_rss_context(
 629         __inout         efx_filter_spec_t *spec,
 630         __in            uint32_t rss_context)
 631 {
 632         efx_rc_t rc;
 633
 634         EFSYS_ASSERT3P(spec, !=, NULL);
 635
 636         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 637         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 638                 rc = EINVAL;
 639                 goto fail1;
 640         }
 641
 642         spec->efs_rss_context = rss_context;
 643
 644         return (0);
 645
 646 fail1:
 647         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 648
 649         return (rc);
 650 }
 651 #endif
 652
 653 #if EFSYS_OPT_SIENA
 654
 655 /*
 656  * "Fudge factors" - difference between programmed value and actual depth.
 657  * Due to pipelined implementation we need to program H/W with a value that
 658  * is larger than the hop limit we want.
 659  */
 660 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 661 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 662
 663 /*
 664  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 665  * We also need to avoid infinite loops in efx_filter_search() when the
 666  * table is full.
 667  */
 668 #define FILTER_CTL_SRCH_MAX 200
 669
 670 static  __checkReturn   efx_rc_t
 671 siena_filter_spec_from_gen_spec(
 672         __out           siena_filter_spec_t *sf_spec,
 673         __in            efx_filter_spec_t *gen_spec)
 674 {
 675         efx_rc_t rc;
 676         boolean_t is_full = B_FALSE;
 677
 678         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 679                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 680         else
 681                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 682
 683         /* Siena only has one RSS context */
 684         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 685             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 686                 rc = EINVAL;
 687                 goto fail1;
 688         }
 689
 690         sf_spec->sfs_flags = gen_spec->efs_flags;
 691         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 692
 693         switch (gen_spec->efs_match_flags) {
 694         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 695             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 696             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 697                 is_full = B_TRUE;
 698                 /* Fall through */
 699         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 700             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 701                 uint32_t rhost, host1, host2;
 702                 uint16_t rport, port1, port2;
 703
 704                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 705                         rc = ENOTSUP;
 706                         goto fail2;
 707                 }
 708                 if (gen_spec->efs_loc_port == 0 ||
 709                     (is_full && gen_spec->efs_rem_port == 0)) {
 710                         rc = EINVAL;
 711                         goto fail3;
 712                 }
 713                 switch (gen_spec->efs_ip_proto) {
 714                 case EFX_IPPROTO_TCP:
 715                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 716                                 sf_spec->sfs_type = (is_full ?
 717                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 718                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 719                         } else {
 720                                 sf_spec->sfs_type = (is_full ?
 721                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 722                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 723                         }
 724                         break;
 725                 case EFX_IPPROTO_UDP:
 726                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 727                                 sf_spec->sfs_type = (is_full ?
 728                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 729                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 730                         } else {
 731                                 sf_spec->sfs_type = (is_full ?
 732                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 733                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 734                         }
 735                         break;
 736                 default:
 737                         rc = ENOTSUP;
 738                         goto fail4;
 739                 }
 740                 /*
 741                  * The filter is constructed in terms of source and destination,
 742                  * with the odd wrinkle that the ports are swapped in a UDP
 743                  * wildcard filter. We need to convert from local and remote
 744                  * addresses (zero for a wildcard).
 745                  */
 746                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 747                 rport = is_full ? gen_spec->efs_rem_port : 0;
 748                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 749                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 750                         host2 = rhost;
 751                 } else {
 752                         host1 = rhost;
 753                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 754                 }
 755                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 756                         if (sf_spec->sfs_type ==
 757                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 758                                 port1 = rport;
 759                                 port2 = gen_spec->efs_loc_port;
 760                         } else {
 761                                 port1 = gen_spec->efs_loc_port;
 762                                 port2 = rport;
 763                         }
 764                 } else {
 765                         if (sf_spec->sfs_type ==
 766                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 767                                 port1 = gen_spec->efs_loc_port;
 768                                 port2 = rport;
 769                         } else {
 770                                 port1 = rport;
 771                                 port2 = gen_spec->efs_loc_port;
 772                         }
 773                 }
 774                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 775                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 776                 sf_spec->sfs_dword[2] = host2;
 777                 break;
 778         }
 779
 780         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 781                 is_full = B_TRUE;
 782                 /* Fall through */
 783         case EFX_FILTER_MATCH_LOC_MAC:
 784                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 785                         sf_spec->sfs_type = (is_full ?
 786                             EFX_SIENA_FILTER_TX_MAC_FULL :
 787                             EFX_SIENA_FILTER_TX_MAC_WILD);
 788                 } else {
 789                         sf_spec->sfs_type = (is_full ?
 790                             EFX_SIENA_FILTER_RX_MAC_FULL :
 791                             EFX_SIENA_FILTER_RX_MAC_WILD);
 792                 }
 793                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 794                 sf_spec->sfs_dword[1] =
 795                     gen_spec->efs_loc_mac[2] << 24 |
 796                     gen_spec->efs_loc_mac[3] << 16 |
 797                     gen_spec->efs_loc_mac[4] <<  8 |
 798                     gen_spec->efs_loc_mac[5];
 799                 sf_spec->sfs_dword[2] =
 800                     gen_spec->efs_loc_mac[0] << 8 |
 801                     gen_spec->efs_loc_mac[1];
 802                 break;
 803
 804         default:
 805                 EFSYS_ASSERT(B_FALSE);
 806                 rc = ENOTSUP;
 807                 goto fail5;
 808         }
 809
 810         return (0);
 811
 812 fail5:
 813         EFSYS_PROBE(fail5);
 814 fail4:
 815         EFSYS_PROBE(fail4);
 816 fail3:
 817         EFSYS_PROBE(fail3);
 818 fail2:
 819         EFSYS_PROBE(fail2);
 820 fail1:
 821         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 822
 823         return (rc);
 824 }
 825
 826 /*
 827  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 828  * key derived from the n-tuple.
 829  */
 830 static                  uint16_t
 831 siena_filter_tbl_hash(
 832         __in            uint32_t key)
 833 {
 834         uint16_t tmp;
 835
 836         /* First 16 rounds */
 837         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 838         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 839         tmp = tmp ^ tmp >> 9;
 840
 841         /* Last 16 rounds */
 842         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 843         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 844         tmp = tmp ^ tmp >> 9;
 845
 846         return (tmp);
 847 }
 848
 849 /*
 850  * To allow for hash collisions, filter search continues at these
 851  * increments from the first possible entry selected by the hash.
 852  */
 853 static                  uint16_t
 854 siena_filter_tbl_increment(
 855         __in            uint32_t key)
 856 {
 857         return ((uint16_t)(key * 2 - 1));
 858 }
 859
 860 static  __checkReturn   boolean_t
 861 siena_filter_test_used(
 862         __in            siena_filter_tbl_t *sftp,
 863         __in            unsigned int index)
 864 {
 865         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 866         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 867 }
 868
 869 static                  void
 870 siena_filter_set_used(
 871         __in            siena_filter_tbl_t *sftp,
 872         __in            unsigned int index)
 873 {
 874         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 875         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 876         ++sftp->sft_used;
 877 }
 878
 879 static                  void
 880 siena_filter_clear_used(
 881         __in            siena_filter_tbl_t *sftp,
 882         __in            unsigned int index)
 883 {
 884         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 885         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 886
 887         --sftp->sft_used;
 888         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 889 }
 890
 891
 892 static                  siena_filter_tbl_id_t
 893 siena_filter_tbl_id(
 894         __in            siena_filter_type_t type)
 895 {
 896         siena_filter_tbl_id_t tbl_id;
 897
 898         switch (type) {
 899         case EFX_SIENA_FILTER_RX_TCP_FULL:
 900         case EFX_SIENA_FILTER_RX_TCP_WILD:
 901         case EFX_SIENA_FILTER_RX_UDP_FULL:
 902         case EFX_SIENA_FILTER_RX_UDP_WILD:
 903                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 904                 break;
 905
 906         case EFX_SIENA_FILTER_RX_MAC_FULL:
 907         case EFX_SIENA_FILTER_RX_MAC_WILD:
 908                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 909                 break;
 910
 911         case EFX_SIENA_FILTER_TX_TCP_FULL:
 912         case EFX_SIENA_FILTER_TX_TCP_WILD:
 913         case EFX_SIENA_FILTER_TX_UDP_FULL:
 914         case EFX_SIENA_FILTER_TX_UDP_WILD:
 915                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 916                 break;
 917
 918         case EFX_SIENA_FILTER_TX_MAC_FULL:
 919         case EFX_SIENA_FILTER_TX_MAC_WILD:
 920                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 921                 break;
 922
 923         default:
 924                 EFSYS_ASSERT(B_FALSE);
 925                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 926                 break;
 927         }
 928         return (tbl_id);
 929 }
 930
 931 static                  void
 932 siena_filter_reset_search_depth(
 933         __inout         siena_filter_t *sfp,
 934         __in            siena_filter_tbl_id_t tbl_id)
 935 {
 936         switch (tbl_id) {
 937         case EFX_SIENA_FILTER_TBL_RX_IP:
 938                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 939                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 940                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 941                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 942                 break;
 943
 944         case EFX_SIENA_FILTER_TBL_RX_MAC:
 945                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 946                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 947                 break;
 948
 949         case EFX_SIENA_FILTER_TBL_TX_IP:
 950                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 951                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 952                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 953                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 954                 break;
 955
 956         case EFX_SIENA_FILTER_TBL_TX_MAC:
 957                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 958                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 959                 break;
 960
 961         default:
 962                 EFSYS_ASSERT(B_FALSE);
 963                 break;
 964         }
 965 }
 966
 967 static                  void
 968 siena_filter_push_rx_limits(
 969         __in            efx_nic_t *enp)
 970 {
 971         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 972         efx_oword_t oword;
 973
 974         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 975
 976         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 977             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 978             FILTER_CTL_SRCH_FUDGE_FULL);
 979         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 980             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 981             FILTER_CTL_SRCH_FUDGE_WILD);
 982         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 983             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 984             FILTER_CTL_SRCH_FUDGE_FULL);
 985         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 986             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 987             FILTER_CTL_SRCH_FUDGE_WILD);
 988
 989         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 990                 EFX_SET_OWORD_FIELD(oword,
 991                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 992                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 993                     FILTER_CTL_SRCH_FUDGE_FULL);
 994                 EFX_SET_OWORD_FIELD(oword,
 995                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 996                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 997                     FILTER_CTL_SRCH_FUDGE_WILD);
 998         }
 999
1000         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
1001 }
1002
1003 static                  void
1004 siena_filter_push_tx_limits(
1005         __in            efx_nic_t *enp)
1006 {
1007         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1008         efx_oword_t oword;
1009
1010         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
1011
1012         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
1013                 EFX_SET_OWORD_FIELD(oword,
1014                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
1015                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
1016                     FILTER_CTL_SRCH_FUDGE_FULL);
1017                 EFX_SET_OWORD_FIELD(oword,
1018                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
1019                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
1020                     FILTER_CTL_SRCH_FUDGE_WILD);
1021                 EFX_SET_OWORD_FIELD(oword,
1022                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
1023                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
1024                     FILTER_CTL_SRCH_FUDGE_FULL);
1025                 EFX_SET_OWORD_FIELD(oword,
1026                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
1027                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
1028                     FILTER_CTL_SRCH_FUDGE_WILD);
1029         }
1030
1031         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
1032                 EFX_SET_OWORD_FIELD(
1033                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
1034                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
1035                         FILTER_CTL_SRCH_FUDGE_FULL);
1036                 EFX_SET_OWORD_FIELD(
1037                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
1038                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
1039                         FILTER_CTL_SRCH_FUDGE_WILD);
1040         }
1041
1042         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
1043 }
1044
1045 /* Build a filter entry and return its n-tuple key. */
1046 static  __checkReturn   uint32_t
1047 siena_filter_build(
1048         __out           efx_oword_t *filter,
1049         __in            siena_filter_spec_t *spec)
1050 {
1051         uint32_t dword3;
1052         uint32_t key;
1053         uint8_t  type  = spec->sfs_type;
1054         uint32_t flags = spec->sfs_flags;
1055
1056         switch (siena_filter_tbl_id(type)) {
1057         case EFX_SIENA_FILTER_TBL_RX_IP: {
1058                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
1059                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
1060                 EFX_POPULATE_OWORD_7(*filter,
1061                     FRF_BZ_RSS_EN,
1062                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1063                     FRF_BZ_SCATTER_EN,
1064                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1065                     FRF_AZ_TCP_UDP, is_udp,
1066                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
1067                     EFX_DWORD_2, spec->sfs_dword[2],
1068                     EFX_DWORD_1, spec->sfs_dword[1],
1069                     EFX_DWORD_0, spec->sfs_dword[0]);
1070                 dword3 = is_udp;
1071                 break;
1072         }
1073
1074         case EFX_SIENA_FILTER_TBL_RX_MAC: {
1075                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
1076                 EFX_POPULATE_OWORD_7(*filter,
1077                     FRF_CZ_RMFT_RSS_EN,
1078                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1079                     FRF_CZ_RMFT_SCATTER_EN,
1080                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1081                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
1082                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
1083                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
1084                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
1085                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
1086                 dword3 = is_wild;
1087                 break;
1088         }
1089
1090         case EFX_SIENA_FILTER_TBL_TX_IP: {
1091                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
1092                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
1093                 EFX_POPULATE_OWORD_5(*filter,
1094                     FRF_CZ_TIFT_TCP_UDP, is_udp,
1095                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
1096                     EFX_DWORD_2, spec->sfs_dword[2],
1097                     EFX_DWORD_1, spec->sfs_dword[1],
1098                     EFX_DWORD_0, spec->sfs_dword[0]);
1099                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
1100                 break;
1101         }
1102
1103         case EFX_SIENA_FILTER_TBL_TX_MAC: {
1104                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
1105                 EFX_POPULATE_OWORD_5(*filter,
1106                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
1107                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
1108                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
1109                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
1110                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
1111                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
1112                 break;
1113         }
1114
1115         default:
1116                 EFSYS_ASSERT(B_FALSE);
1117                 EFX_ZERO_OWORD(*filter);
1118                 return (0);
1119         }
1120
1121         key =
1122             spec->sfs_dword[0] ^
1123             spec->sfs_dword[1] ^
1124             spec->sfs_dword[2] ^
1125             dword3;
1126
1127         return (key);
1128 }
1129
1130 static  __checkReturn           efx_rc_t
1131 siena_filter_push_entry(
1132         __inout                 efx_nic_t *enp,
1133         __in                    siena_filter_type_t type,
1134         __in                    int index,
1135         __in                    efx_oword_t *eop)
1136 {
1137         efx_rc_t rc;
1138
1139         switch (type) {
1140         case EFX_SIENA_FILTER_RX_TCP_FULL:
1141         case EFX_SIENA_FILTER_RX_TCP_WILD:
1142         case EFX_SIENA_FILTER_RX_UDP_FULL:
1143         case EFX_SIENA_FILTER_RX_UDP_WILD:
1144                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1145                     eop, B_TRUE);
1146                 break;
1147
1148         case EFX_SIENA_FILTER_RX_MAC_FULL:
1149         case EFX_SIENA_FILTER_RX_MAC_WILD:
1150                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1151                     eop, B_TRUE);
1152                 break;
1153
1154         case EFX_SIENA_FILTER_TX_TCP_FULL:
1155         case EFX_SIENA_FILTER_TX_TCP_WILD:
1156         case EFX_SIENA_FILTER_TX_UDP_FULL:
1157         case EFX_SIENA_FILTER_TX_UDP_WILD:
1158                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1159                     eop, B_TRUE);
1160                 break;
1161
1162         case EFX_SIENA_FILTER_TX_MAC_FULL:
1163         case EFX_SIENA_FILTER_TX_MAC_WILD:
1164                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1165                     eop, B_TRUE);
1166                 break;
1167
1168         default:
1169                 EFSYS_ASSERT(B_FALSE);
1170                 rc = ENOTSUP;
1171                 goto fail1;
1172         }
1173         return (0);
1174
1175 fail1:
1176         return (rc);
1177 }
1178
1179
1180 static  __checkReturn   boolean_t
1181 siena_filter_equal(
1182         __in            const siena_filter_spec_t *left,
1183         __in            const siena_filter_spec_t *right)
1184 {
1185         siena_filter_tbl_id_t tbl_id;
1186
1187         tbl_id = siena_filter_tbl_id(left->sfs_type);
1188
1189
1190         if (left->sfs_type != right->sfs_type)
1191                 return (B_FALSE);
1192
1193         if (memcmp(left->sfs_dword, right->sfs_dword,
1194                 sizeof (left->sfs_dword)))
1195                 return (B_FALSE);
1196
1197         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1198                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1199             left->sfs_dmaq_id != right->sfs_dmaq_id)
1200                 return (B_FALSE);
1201
1202         return (B_TRUE);
1203 }
1204
1205 static  __checkReturn   efx_rc_t
1206 siena_filter_search(
1207         __in            siena_filter_tbl_t *sftp,
1208         __in            siena_filter_spec_t *spec,
1209         __in            uint32_t key,
1210         __in            boolean_t for_insert,
1211         __out           int *filter_index,
1212         __out           unsigned int *depth_required)
1213 {
1214         unsigned int hash, incr, filter_idx, depth;
1215
1216         hash = siena_filter_tbl_hash(key);
1217         incr = siena_filter_tbl_increment(key);
1218
1219         filter_idx = hash & (sftp->sft_size - 1);
1220         depth = 1;
1221
1222         for (;;) {
1223                 /*
1224                  * Return success if entry is used and matches this spec
1225                  * or entry is unused and we are trying to insert.
1226                  */
1227                 if (siena_filter_test_used(sftp, filter_idx) ?
1228                     siena_filter_equal(spec,
1229                     &sftp->sft_spec[filter_idx]) :
1230                     for_insert) {
1231                         *filter_index = filter_idx;
1232                         *depth_required = depth;
1233                         return (0);
1234                 }
1235
1236                 /* Return failure if we reached the maximum search depth */
1237                 if (depth == FILTER_CTL_SRCH_MAX)
1238                         return (for_insert ? EBUSY : ENOENT);
1239
1240                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1241                 ++depth;
1242         }
1243 }
1244
1245 static                  void
1246 siena_filter_clear_entry(
1247         __in            efx_nic_t *enp,
1248         __in            siena_filter_tbl_t *sftp,
1249         __in            int index)
1250 {
1251         efx_oword_t filter;
1252
1253         if (siena_filter_test_used(sftp, index)) {
1254                 siena_filter_clear_used(sftp, index);
1255
1256                 EFX_ZERO_OWORD(filter);
1257                 siena_filter_push_entry(enp,
1258                     sftp->sft_spec[index].sfs_type,
1259                     index, &filter);
1260
1261                 memset(&sftp->sft_spec[index],
1262                     0, sizeof (sftp->sft_spec[0]));
1263         }
1264 }
1265
1266                         void
1267 siena_filter_tbl_clear(
1268         __in            efx_nic_t *enp,
1269         __in            siena_filter_tbl_id_t tbl_id)
1270 {
1271         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1272         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1273         int index;
1274         efsys_lock_state_t state;
1275
1276         EFSYS_LOCK(enp->en_eslp, state);
1277
1278         for (index = 0; index < sftp->sft_size; ++index) {
1279                 siena_filter_clear_entry(enp, sftp, index);
1280         }
1281
1282         if (sftp->sft_used == 0)
1283                 siena_filter_reset_search_depth(sfp, tbl_id);
1284
1285         EFSYS_UNLOCK(enp->en_eslp, state);
1286 }
1287
1288 static  __checkReturn   efx_rc_t
1289 siena_filter_init(
1290         __in            efx_nic_t *enp)
1291 {
1292         siena_filter_t *sfp;
1293         siena_filter_tbl_t *sftp;
1294         int tbl_id;
1295         efx_rc_t rc;
1296
1297         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1298
1299         if (!sfp) {
1300                 rc = ENOMEM;
1301                 goto fail1;
1302         }
1303
1304         enp->en_filter.ef_siena_filter = sfp;
1305
1306         switch (enp->en_family) {
1307         case EFX_FAMILY_SIENA:
1308                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1309                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1310
1311                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1312                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1313
1314                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1315                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1316
1317                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1318                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1319                 break;
1320
1321         default:
1322                 rc = ENOTSUP;
1323                 goto fail2;
1324         }
1325
1326         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1327                 unsigned int bitmap_size;
1328
1329                 sftp = &sfp->sf_tbl[tbl_id];
1330                 if (sftp->sft_size == 0)
1331                         continue;
1332
1333                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1334                     sizeof (uint32_t));
1335                 bitmap_size =
1336                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1337
1338                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1339                 if (!sftp->sft_bitmap) {
1340                         rc = ENOMEM;
1341                         goto fail3;
1342                 }
1343
1344                 EFSYS_KMEM_ALLOC(enp->en_esip,
1345                     sftp->sft_size * sizeof (*sftp->sft_spec),
1346                     sftp->sft_spec);
1347                 if (!sftp->sft_spec) {
1348                         rc = ENOMEM;
1349                         goto fail4;
1350                 }
1351                 memset(sftp->sft_spec, 0,
1352                     sftp->sft_size * sizeof (*sftp->sft_spec));
1353         }
1354
1355         return (0);
1356
1357 fail4:
1358         EFSYS_PROBE(fail4);
1359
1360 fail3:
1361         EFSYS_PROBE(fail3);
1362
1363 fail2:
1364         EFSYS_PROBE(fail2);
1365         siena_filter_fini(enp);
1366
1367 fail1:
1368         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1369         return (rc);
1370 }
1371
1372 static                  void
1373 siena_filter_fini(
1374         __in            efx_nic_t *enp)
1375 {
1376         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1377         siena_filter_tbl_id_t tbl_id;
1378
1379         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1380         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1381
1382         if (sfp == NULL)
1383                 return;
1384
1385         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1386                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1387                 unsigned int bitmap_size;
1388
1389                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1390                     sizeof (uint32_t));
1391                 bitmap_size =
1392                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1393
1394                 if (sftp->sft_bitmap != NULL) {
1395                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1396                             sftp->sft_bitmap);
1397                         sftp->sft_bitmap = NULL;
1398                 }
1399
1400                 if (sftp->sft_spec != NULL) {
1401                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1402                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1403                         sftp->sft_spec = NULL;
1404                 }
1405         }
1406
1407         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1408             enp->en_filter.ef_siena_filter);
1409 }
1410
1411 /* Restore filter state after a reset */
1412 static  __checkReturn   efx_rc_t
1413 siena_filter_restore(
1414         __in            efx_nic_t *enp)
1415 {
1416         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1417         siena_filter_tbl_id_t tbl_id;
1418         siena_filter_tbl_t *sftp;
1419         siena_filter_spec_t *spec;
1420         efx_oword_t filter;
1421         int filter_idx;
1422         efsys_lock_state_t state;
1423         uint32_t key;
1424         efx_rc_t rc;
1425
1426         EFSYS_LOCK(enp->en_eslp, state);
1427
1428         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1429                 sftp = &sfp->sf_tbl[tbl_id];
1430                 for (filter_idx = 0;
1431                         filter_idx < sftp->sft_size;
1432                         filter_idx++) {
1433                         if (!siena_filter_test_used(sftp, filter_idx))
1434                                 continue;
1435
1436                         spec = &sftp->sft_spec[filter_idx];
1437                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1438                                 rc = EINVAL;
1439                                 goto fail1;
1440                         }
1441                         if ((rc = siena_filter_push_entry(enp,
1442                                     spec->sfs_type, filter_idx, &filter)) != 0)
1443                                 goto fail2;
1444                 }
1445         }
1446
1447         siena_filter_push_rx_limits(enp);
1448         siena_filter_push_tx_limits(enp);
1449
1450         EFSYS_UNLOCK(enp->en_eslp, state);
1451
1452         return (0);
1453
1454 fail2:
1455         EFSYS_PROBE(fail2);
1456
1457 fail1:
1458         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1459
1460         EFSYS_UNLOCK(enp->en_eslp, state);
1461
1462         return (rc);
1463 }
1464
1465 static   __checkReturn  efx_rc_t
1466 siena_filter_add(
1467         __in            efx_nic_t *enp,
1468         __inout         efx_filter_spec_t *spec,
1469         __in            boolean_t may_replace)
1470 {
1471         efx_rc_t rc;
1472         siena_filter_spec_t sf_spec;
1473         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1474         siena_filter_tbl_id_t tbl_id;
1475         siena_filter_tbl_t *sftp;
1476         siena_filter_spec_t *saved_sf_spec;
1477         efx_oword_t filter;
1478         int filter_idx;
1479         unsigned int depth;
1480         efsys_lock_state_t state;
1481         uint32_t key;
1482
1483
1484         EFSYS_ASSERT3P(spec, !=, NULL);
1485
1486         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1487                 goto fail1;
1488
1489         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1490         sftp = &sfp->sf_tbl[tbl_id];
1491
1492         if (sftp->sft_size == 0) {
1493                 rc = EINVAL;
1494                 goto fail2;
1495         }
1496
1497         key = siena_filter_build(&filter, &sf_spec);
1498
1499         EFSYS_LOCK(enp->en_eslp, state);
1500
1501         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1502             &filter_idx, &depth);
1503         if (rc != 0)
1504                 goto fail3;
1505
1506         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1507         saved_sf_spec = &sftp->sft_spec[filter_idx];
1508
1509         if (siena_filter_test_used(sftp, filter_idx)) {
1510                 if (may_replace == B_FALSE) {
1511                         rc = EEXIST;
1512                         goto fail4;
1513                 }
1514         }
1515         siena_filter_set_used(sftp, filter_idx);
1516         *saved_sf_spec = sf_spec;
1517
1518         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1519                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1520                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1521                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1522                         siena_filter_push_tx_limits(enp);
1523                 else
1524                         siena_filter_push_rx_limits(enp);
1525         }
1526
1527         siena_filter_push_entry(enp, sf_spec.sfs_type,
1528             filter_idx, &filter);
1529
1530         EFSYS_UNLOCK(enp->en_eslp, state);
1531         return (0);
1532
1533 fail4:
1534         EFSYS_PROBE(fail4);
1535
1536 fail3:
1537         EFSYS_UNLOCK(enp->en_eslp, state);
1538         EFSYS_PROBE(fail3);
1539
1540 fail2:
1541         EFSYS_PROBE(fail2);
1542
1543 fail1:
1544         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1545         return (rc);
1546 }
1547
1548 static   __checkReturn  efx_rc_t
1549 siena_filter_delete(
1550         __in            efx_nic_t *enp,
1551         __inout         efx_filter_spec_t *spec)
1552 {
1553         efx_rc_t rc;
1554         siena_filter_spec_t sf_spec;
1555         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1556         siena_filter_tbl_id_t tbl_id;
1557         siena_filter_tbl_t *sftp;
1558         efx_oword_t filter;
1559         int filter_idx;
1560         unsigned int depth;
1561         efsys_lock_state_t state;
1562         uint32_t key;
1563
1564         EFSYS_ASSERT3P(spec, !=, NULL);
1565
1566         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1567                 goto fail1;
1568
1569         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1570         sftp = &sfp->sf_tbl[tbl_id];
1571
1572         key = siena_filter_build(&filter, &sf_spec);
1573
1574         EFSYS_LOCK(enp->en_eslp, state);
1575
1576         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1577             &filter_idx, &depth);
1578         if (rc != 0)
1579                 goto fail2;
1580
1581         siena_filter_clear_entry(enp, sftp, filter_idx);
1582         if (sftp->sft_used == 0)
1583                 siena_filter_reset_search_depth(sfp, tbl_id);
1584
1585         EFSYS_UNLOCK(enp->en_eslp, state);
1586         return (0);
1587
1588 fail2:
1589         EFSYS_UNLOCK(enp->en_eslp, state);
1590         EFSYS_PROBE(fail2);
1591
1592 fail1:
1593         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1594         return (rc);
1595 }
1596
1597 #define SIENA_MAX_SUPPORTED_MATCHES 4
1598
1599 static  __checkReturn   efx_rc_t
1600 siena_filter_supported_filters(
1601         __in                            efx_nic_t *enp,
1602         __out_ecount(buffer_length)     uint32_t *buffer,
1603         __in                            size_t buffer_length,
1604         __out                           size_t *list_lengthp)
1605 {
1606         uint32_t index = 0;
1607         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1608         size_t list_length;
1609         efx_rc_t rc;
1610
1611         rx_matches[index++] =
1612             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1613             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1614             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1615
1616         rx_matches[index++] =
1617             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1618             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1619
1620         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1621                 rx_matches[index++] =
1622                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1623
1624                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1625         }
1626
1627         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1628         list_length = index;
1629
1630         *list_lengthp = list_length;
1631
1632         if (buffer_length < list_length) {
1633                 rc = ENOSPC;
1634                 goto fail1;
1635         }
1636
1637         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1638
1639         return (0);
1640
1641 fail1:
1642         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1643
1644         return (rc);
1645 }
1646
1647 #undef MAX_SUPPORTED
1648
1649 #endif /* EFSYS_OPT_SIENA */
1650
1651 #endif /* EFSYS_OPT_FILTER */