sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include "efx.h"
  37 #include "efx_impl.h"
  38
  39
  40 #if EFSYS_OPT_FILTER
  41
  42 #if EFSYS_OPT_SIENA
  43
  44 static  __checkReturn   efx_rc_t
  45 siena_filter_init(
  46         __in            efx_nic_t *enp);
  47
  48 static                  void
  49 siena_filter_fini(
  50         __in            efx_nic_t *enp);
  51
  52 static  __checkReturn   efx_rc_t
  53 siena_filter_restore(
  54         __in            efx_nic_t *enp);
  55
  56 static  __checkReturn   efx_rc_t
  57 siena_filter_add(
  58         __in            efx_nic_t *enp,
  59         __inout         efx_filter_spec_t *spec,
  60         __in            boolean_t may_replace);
  61
  62 static  __checkReturn   efx_rc_t
  63 siena_filter_delete(
  64         __in            efx_nic_t *enp,
  65         __inout         efx_filter_spec_t *spec);
  66
  67 static  __checkReturn   efx_rc_t
  68 siena_filter_supported_filters(
  69         __in                            efx_nic_t *enp,
  70         __out_ecount(buffer_length)     uint32_t *buffer,
  71         __in                            size_t buffer_length,
  72         __out                           size_t *list_lengthp);
  73
  74 #endif /* EFSYS_OPT_SIENA */
  75
  76 #if EFSYS_OPT_SIENA
  77 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  78         siena_filter_init,              /* efo_init */
  79         siena_filter_fini,              /* efo_fini */
  80         siena_filter_restore,           /* efo_restore */
  81         siena_filter_add,               /* efo_add */
  82         siena_filter_delete,            /* efo_delete */
  83         siena_filter_supported_filters, /* efo_supported_filters */
  84         NULL,                           /* efo_reconfigure */
  85 };
  86 #endif /* EFSYS_OPT_SIENA */
  87
  88 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  89 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  90         ef10_filter_init,               /* efo_init */
  91         ef10_filter_fini,               /* efo_fini */
  92         ef10_filter_restore,            /* efo_restore */
  93         ef10_filter_add,                /* efo_add */
  94         ef10_filter_delete,             /* efo_delete */
  95         ef10_filter_supported_filters,  /* efo_supported_filters */
  96         ef10_filter_reconfigure,        /* efo_reconfigure */
  97 };
  98 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
  99
 100         __checkReturn   efx_rc_t
 101 efx_filter_insert(
 102         __in            efx_nic_t *enp,
 103         __inout         efx_filter_spec_t *spec)
 104 {
 105         const efx_filter_ops_t *efop = enp->en_efop;
 106         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 107         efx_rc_t rc;
 108
 109         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 110         EFSYS_ASSERT3P(spec, !=, NULL);
 111         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 112
 113         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) &&
 114             !encp->enc_filter_action_mark_supported) {
 115                 rc = ENOTSUP;
 116                 goto fail1;
 117         }
 118
 119         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG) &&
 120             !encp->enc_filter_action_flag_supported) {
 121                 rc = ENOTSUP;
 122                 goto fail2;
 123         }
 124
 125         return (efop->efo_add(enp, spec, B_FALSE));
 126
 127 fail2:
 128         EFSYS_PROBE(fail2);
 129 fail1:
 130         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 131
 132         return (rc);
 133 }
 134
 135         __checkReturn   efx_rc_t
 136 efx_filter_remove(
 137         __in            efx_nic_t *enp,
 138         __inout         efx_filter_spec_t *spec)
 139 {
 140         const efx_filter_ops_t *efop = enp->en_efop;
 141
 142         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 143         EFSYS_ASSERT3P(spec, !=, NULL);
 144         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 145
 146         return (efop->efo_delete(enp, spec));
 147 }
 148
 149         __checkReturn   efx_rc_t
 150 efx_filter_restore(
 151         __in            efx_nic_t *enp)
 152 {
 153         efx_rc_t rc;
 154
 155         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 156
 157         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 158                 goto fail1;
 159
 160         return (0);
 161
 162 fail1:
 163         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 164
 165         return (rc);
 166 }
 167
 168         __checkReturn   efx_rc_t
 169 efx_filter_init(
 170         __in            efx_nic_t *enp)
 171 {
 172         const efx_filter_ops_t *efop;
 173         efx_rc_t rc;
 174
 175         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 176         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 177         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 178
 179         switch (enp->en_family) {
 180 #if EFSYS_OPT_SIENA
 181         case EFX_FAMILY_SIENA:
 182                 efop = &__efx_filter_siena_ops;
 183                 break;
 184 #endif /* EFSYS_OPT_SIENA */
 185
 186 #if EFSYS_OPT_HUNTINGTON
 187         case EFX_FAMILY_HUNTINGTON:
 188                 efop = &__efx_filter_ef10_ops;
 189                 break;
 190 #endif /* EFSYS_OPT_HUNTINGTON */
 191
 192 #if EFSYS_OPT_MEDFORD
 193         case EFX_FAMILY_MEDFORD:
 194                 efop = &__efx_filter_ef10_ops;
 195                 break;
 196 #endif /* EFSYS_OPT_MEDFORD */
 197
 198 #if EFSYS_OPT_MEDFORD2
 199         case EFX_FAMILY_MEDFORD2:
 200                 efop = &__efx_filter_ef10_ops;
 201                 break;
 202 #endif /* EFSYS_OPT_MEDFORD2 */
 203
 204         default:
 205                 EFSYS_ASSERT(0);
 206                 rc = ENOTSUP;
 207                 goto fail1;
 208         }
 209
 210         if ((rc = efop->efo_init(enp)) != 0)
 211                 goto fail2;
 212
 213         enp->en_efop = efop;
 214         enp->en_mod_flags |= EFX_MOD_FILTER;
 215         return (0);
 216
 217 fail2:
 218         EFSYS_PROBE(fail2);
 219 fail1:
 220         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 221
 222         enp->en_efop = NULL;
 223         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 224         return (rc);
 225 }
 226
 227                         void
 228 efx_filter_fini(
 229         __in            efx_nic_t *enp)
 230 {
 231         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 232         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 233         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 234
 235         enp->en_efop->efo_fini(enp);
 236
 237         enp->en_efop = NULL;
 238         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 239 }
 240
 241 /*
 242  * Query the possible combinations of match flags which can be filtered on.
 243  * These are returned as a list, of which each 32 bit element is a bitmask
 244  * formed of EFX_FILTER_MATCH flags.
 245  *
 246  * The combinations are ordered in priority from highest to lowest.
 247  *
 248  * If the provided buffer is too short to hold the list, the call with fail with
 249  * ENOSPC and *list_lengthp will be set to the buffer length required.
 250  */
 251         __checkReturn   efx_rc_t
 252 efx_filter_supported_filters(
 253         __in                            efx_nic_t *enp,
 254         __out_ecount(buffer_length)     uint32_t *buffer,
 255         __in                            size_t buffer_length,
 256         __out                           size_t *list_lengthp)
 257 {
 258         efx_rc_t rc;
 259
 260         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 261         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 262         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 263         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 264
 265         if (buffer == NULL) {
 266                 rc = EINVAL;
 267                 goto fail1;
 268         }
 269
 270         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 271                                                     list_lengthp);
 272         if (rc != 0)
 273                 goto fail2;
 274
 275         return (0);
 276
 277 fail2:
 278         EFSYS_PROBE(fail2);
 279 fail1:
 280         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 281
 282         return (rc);
 283 }
 284
 285         __checkReturn   efx_rc_t
 286 efx_filter_reconfigure(
 287         __in                            efx_nic_t *enp,
 288         __in_ecount(6)                  uint8_t const *mac_addr,
 289         __in                            boolean_t all_unicst,
 290         __in                            boolean_t mulcst,
 291         __in                            boolean_t all_mulcst,
 292         __in                            boolean_t brdcst,
 293         __in_ecount(6*count)            uint8_t const *addrs,
 294         __in                            uint32_t count)
 295 {
 296         efx_rc_t rc;
 297
 298         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 299         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 300         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 301
 302         if (enp->en_efop->efo_reconfigure != NULL) {
 303                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 304                                                         all_unicst, mulcst,
 305                                                         all_mulcst, brdcst,
 306                                                         addrs, count)) != 0)
 307                         goto fail1;
 308         }
 309
 310         return (0);
 311
 312 fail1:
 313         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 314
 315         return (rc);
 316 }
 317
 318                 void
 319 efx_filter_spec_init_rx(
 320         __out           efx_filter_spec_t *spec,
 321         __in            efx_filter_priority_t priority,
 322         __in            efx_filter_flags_t flags,
 323         __in            efx_rxq_t *erp)
 324 {
 325         EFSYS_ASSERT3P(spec, !=, NULL);
 326         EFSYS_ASSERT3P(erp, !=, NULL);
 327         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 328                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 329
 330         memset(spec, 0, sizeof (*spec));
 331         spec->efs_priority = priority;
 332         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 333         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 334         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 335 }
 336
 337                 void
 338 efx_filter_spec_init_tx(
 339         __out           efx_filter_spec_t *spec,
 340         __in            efx_txq_t *etp)
 341 {
 342         EFSYS_ASSERT3P(spec, !=, NULL);
 343         EFSYS_ASSERT3P(etp, !=, NULL);
 344
 345         memset(spec, 0, sizeof (*spec));
 346         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 347         spec->efs_flags = EFX_FILTER_FLAG_TX;
 348         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 349 }
 350
 351
 352 /*
 353  *  Specify IPv4 host, transport protocol and port in a filter specification
 354  */
 355 __checkReturn           efx_rc_t
 356 efx_filter_spec_set_ipv4_local(
 357         __inout         efx_filter_spec_t *spec,
 358         __in            uint8_t proto,
 359         __in            uint32_t host,
 360         __in            uint16_t port)
 361 {
 362         EFSYS_ASSERT3P(spec, !=, NULL);
 363
 364         spec->efs_match_flags |=
 365                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 366                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 367         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 368         spec->efs_ip_proto = proto;
 369         spec->efs_loc_host.eo_u32[0] = host;
 370         spec->efs_loc_port = port;
 371         return (0);
 372 }
 373
 374 /*
 375  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 376  */
 377 __checkReturn           efx_rc_t
 378 efx_filter_spec_set_ipv4_full(
 379         __inout         efx_filter_spec_t *spec,
 380         __in            uint8_t proto,
 381         __in            uint32_t lhost,
 382         __in            uint16_t lport,
 383         __in            uint32_t rhost,
 384         __in            uint16_t rport)
 385 {
 386         EFSYS_ASSERT3P(spec, !=, NULL);
 387
 388         spec->efs_match_flags |=
 389                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 390                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 391                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 392         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 393         spec->efs_ip_proto = proto;
 394         spec->efs_loc_host.eo_u32[0] = lhost;
 395         spec->efs_loc_port = lport;
 396         spec->efs_rem_host.eo_u32[0] = rhost;
 397         spec->efs_rem_port = rport;
 398         return (0);
 399 }
 400
 401 /*
 402  * Specify local Ethernet address and/or VID in filter specification
 403  */
 404 __checkReturn           efx_rc_t
 405 efx_filter_spec_set_eth_local(
 406         __inout         efx_filter_spec_t *spec,
 407         __in            uint16_t vid,
 408         __in            const uint8_t *addr)
 409 {
 410         EFSYS_ASSERT3P(spec, !=, NULL);
 411         EFSYS_ASSERT3P(addr, !=, NULL);
 412
 413         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 414                 return (EINVAL);
 415
 416         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 417                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 418                 spec->efs_outer_vid = vid;
 419         }
 420         if (addr != NULL) {
 421                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 422                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 423         }
 424         return (0);
 425 }
 426
 427                         void
 428 efx_filter_spec_set_ether_type(
 429         __inout         efx_filter_spec_t *spec,
 430         __in            uint16_t ether_type)
 431 {
 432         EFSYS_ASSERT3P(spec, !=, NULL);
 433
 434         spec->efs_ether_type = ether_type;
 435         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 436 }
 437
 438 /*
 439  * Specify matching otherwise-unmatched unicast in a filter specification
 440  */
 441 __checkReturn           efx_rc_t
 442 efx_filter_spec_set_uc_def(
 443         __inout         efx_filter_spec_t *spec)
 444 {
 445         EFSYS_ASSERT3P(spec, !=, NULL);
 446
 447         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 448         return (0);
 449 }
 450
 451 /*
 452  * Specify matching otherwise-unmatched multicast in a filter specification
 453  */
 454 __checkReturn           efx_rc_t
 455 efx_filter_spec_set_mc_def(
 456         __inout         efx_filter_spec_t *spec)
 457 {
 458         EFSYS_ASSERT3P(spec, !=, NULL);
 459
 460         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 461         return (0);
 462 }
 463
 464
 465 __checkReturn           efx_rc_t
 466 efx_filter_spec_set_encap_type(
 467         __inout         efx_filter_spec_t *spec,
 468         __in            efx_tunnel_protocol_t encap_type,
 469         __in            efx_filter_inner_frame_match_t inner_frame_match)
 470 {
 471         uint32_t match_flags = EFX_FILTER_MATCH_ENCAP_TYPE;
 472         uint8_t ip_proto;
 473         efx_rc_t rc;
 474
 475         EFSYS_ASSERT3P(spec, !=, NULL);
 476
 477         switch (encap_type) {
 478         case EFX_TUNNEL_PROTOCOL_VXLAN:
 479         case EFX_TUNNEL_PROTOCOL_GENEVE:
 480                 ip_proto = EFX_IPPROTO_UDP;
 481                 break;
 482         case EFX_TUNNEL_PROTOCOL_NVGRE:
 483                 ip_proto = EFX_IPPROTO_GRE;
 484                 break;
 485         default:
 486                 EFSYS_ASSERT(0);
 487                 rc = EINVAL;
 488                 goto fail1;
 489         }
 490
 491         switch (inner_frame_match) {
 492         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 493                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 494                 break;
 495         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 496                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 497                 break;
 498         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 499                 /* This is for when specific inner frames are to be matched. */
 500                 break;
 501         default:
 502                 EFSYS_ASSERT(0);
 503                 rc = EINVAL;
 504                 goto fail2;
 505         }
 506
 507         spec->efs_encap_type = encap_type;
 508         spec->efs_ip_proto = ip_proto;
 509         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 510
 511         return (0);
 512
 513 fail2:
 514         EFSYS_PROBE(fail2);
 515 fail1:
 516         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 517
 518         return (rc);
 519 }
 520
 521 /*
 522  * Specify inner and outer Ethernet address and VXLAN ID in filter
 523  * specification.
 524  */
 525         __checkReturn   efx_rc_t
 526 efx_filter_spec_set_vxlan_full(
 527         __inout         efx_filter_spec_t *spec,
 528         __in            const uint8_t *vxlan_id,
 529         __in            const uint8_t *inner_addr,
 530         __in            const uint8_t *outer_addr)
 531 {
 532         EFSYS_ASSERT3P(spec, !=, NULL);
 533         EFSYS_ASSERT3P(vxlan_id, !=, NULL);
 534         EFSYS_ASSERT3P(inner_addr, !=, NULL);
 535         EFSYS_ASSERT3P(outer_addr, !=, NULL);
 536
 537         if ((inner_addr == NULL) && (outer_addr == NULL))
 538                 return (EINVAL);
 539
 540         if (vxlan_id != NULL) {
 541                 spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 542                 memcpy(spec->efs_vni_or_vsid, vxlan_id, EFX_VNI_OR_VSID_LEN);
 543         }
 544         if (outer_addr != NULL) {
 545                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 546                 memcpy(spec->efs_loc_mac, outer_addr, EFX_MAC_ADDR_LEN);
 547         }
 548         if (inner_addr != NULL) {
 549                 spec->efs_match_flags |= EFX_FILTER_MATCH_IFRM_LOC_MAC;
 550                 memcpy(spec->efs_ifrm_loc_mac, inner_addr, EFX_MAC_ADDR_LEN);
 551         }
 552         spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 553         spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_VXLAN;
 554
 555         return (0);
 556 }
 557
 558 #if EFSYS_OPT_RX_SCALE
 559         __checkReturn   efx_rc_t
 560 efx_filter_spec_set_rss_context(
 561         __inout         efx_filter_spec_t *spec,
 562         __in            uint32_t rss_context)
 563 {
 564         efx_rc_t rc;
 565
 566         EFSYS_ASSERT3P(spec, !=, NULL);
 567
 568         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 569         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 570                 rc = EINVAL;
 571                 goto fail1;
 572         }
 573
 574         spec->efs_rss_context = rss_context;
 575
 576         return (0);
 577
 578 fail1:
 579         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 580
 581         return (rc);
 582 }
 583 #endif
 584
 585 #if EFSYS_OPT_SIENA
 586
 587 /*
 588  * "Fudge factors" - difference between programmed value and actual depth.
 589  * Due to pipelined implementation we need to program H/W with a value that
 590  * is larger than the hop limit we want.
 591  */
 592 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 593 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 594
 595 /*
 596  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 597  * We also need to avoid infinite loops in efx_filter_search() when the
 598  * table is full.
 599  */
 600 #define FILTER_CTL_SRCH_MAX 200
 601
 602 static  __checkReturn   efx_rc_t
 603 siena_filter_spec_from_gen_spec(
 604         __out           siena_filter_spec_t *sf_spec,
 605         __in            efx_filter_spec_t *gen_spec)
 606 {
 607         efx_rc_t rc;
 608         boolean_t is_full = B_FALSE;
 609
 610         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 611                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 612         else
 613                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 614
 615         /* Siena only has one RSS context */
 616         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 617             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 618                 rc = EINVAL;
 619                 goto fail1;
 620         }
 621
 622         sf_spec->sfs_flags = gen_spec->efs_flags;
 623         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 624
 625         switch (gen_spec->efs_match_flags) {
 626         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 627             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 628             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 629                 is_full = B_TRUE;
 630                 /* Fall through */
 631         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 632             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 633                 uint32_t rhost, host1, host2;
 634                 uint16_t rport, port1, port2;
 635
 636                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 637                         rc = ENOTSUP;
 638                         goto fail2;
 639                 }
 640                 if (gen_spec->efs_loc_port == 0 ||
 641                     (is_full && gen_spec->efs_rem_port == 0)) {
 642                         rc = EINVAL;
 643                         goto fail3;
 644                 }
 645                 switch (gen_spec->efs_ip_proto) {
 646                 case EFX_IPPROTO_TCP:
 647                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 648                                 sf_spec->sfs_type = (is_full ?
 649                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 650                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 651                         } else {
 652                                 sf_spec->sfs_type = (is_full ?
 653                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 654                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 655                         }
 656                         break;
 657                 case EFX_IPPROTO_UDP:
 658                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 659                                 sf_spec->sfs_type = (is_full ?
 660                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 661                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 662                         } else {
 663                                 sf_spec->sfs_type = (is_full ?
 664                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 665                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 666                         }
 667                         break;
 668                 default:
 669                         rc = ENOTSUP;
 670                         goto fail4;
 671                 }
 672                 /*
 673                  * The filter is constructed in terms of source and destination,
 674                  * with the odd wrinkle that the ports are swapped in a UDP
 675                  * wildcard filter. We need to convert from local and remote
 676                  * addresses (zero for a wildcard).
 677                  */
 678                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 679                 rport = is_full ? gen_spec->efs_rem_port : 0;
 680                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 681                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 682                         host2 = rhost;
 683                 } else {
 684                         host1 = rhost;
 685                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 686                 }
 687                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 688                         if (sf_spec->sfs_type ==
 689                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 690                                 port1 = rport;
 691                                 port2 = gen_spec->efs_loc_port;
 692                         } else {
 693                                 port1 = gen_spec->efs_loc_port;
 694                                 port2 = rport;
 695                         }
 696                 } else {
 697                         if (sf_spec->sfs_type ==
 698                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 699                                 port1 = gen_spec->efs_loc_port;
 700                                 port2 = rport;
 701                         } else {
 702                                 port1 = rport;
 703                                 port2 = gen_spec->efs_loc_port;
 704                         }
 705                 }
 706                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 707                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 708                 sf_spec->sfs_dword[2] = host2;
 709                 break;
 710         }
 711
 712         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 713                 is_full = B_TRUE;
 714                 /* Fall through */
 715         case EFX_FILTER_MATCH_LOC_MAC:
 716                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 717                         sf_spec->sfs_type = (is_full ?
 718                             EFX_SIENA_FILTER_TX_MAC_FULL :
 719                             EFX_SIENA_FILTER_TX_MAC_WILD);
 720                 } else {
 721                         sf_spec->sfs_type = (is_full ?
 722                             EFX_SIENA_FILTER_RX_MAC_FULL :
 723                             EFX_SIENA_FILTER_RX_MAC_WILD);
 724                 }
 725                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 726                 sf_spec->sfs_dword[1] =
 727                     gen_spec->efs_loc_mac[2] << 24 |
 728                     gen_spec->efs_loc_mac[3] << 16 |
 729                     gen_spec->efs_loc_mac[4] <<  8 |
 730                     gen_spec->efs_loc_mac[5];
 731                 sf_spec->sfs_dword[2] =
 732                     gen_spec->efs_loc_mac[0] << 8 |
 733                     gen_spec->efs_loc_mac[1];
 734                 break;
 735
 736         default:
 737                 EFSYS_ASSERT(B_FALSE);
 738                 rc = ENOTSUP;
 739                 goto fail5;
 740         }
 741
 742         return (0);
 743
 744 fail5:
 745         EFSYS_PROBE(fail5);
 746 fail4:
 747         EFSYS_PROBE(fail4);
 748 fail3:
 749         EFSYS_PROBE(fail3);
 750 fail2:
 751         EFSYS_PROBE(fail2);
 752 fail1:
 753         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 754
 755         return (rc);
 756 }
 757
 758 /*
 759  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 760  * key derived from the n-tuple.
 761  */
 762 static                  uint16_t
 763 siena_filter_tbl_hash(
 764         __in            uint32_t key)
 765 {
 766         uint16_t tmp;
 767
 768         /* First 16 rounds */
 769         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 770         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 771         tmp = tmp ^ tmp >> 9;
 772
 773         /* Last 16 rounds */
 774         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 775         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 776         tmp = tmp ^ tmp >> 9;
 777
 778         return (tmp);
 779 }
 780
 781 /*
 782  * To allow for hash collisions, filter search continues at these
 783  * increments from the first possible entry selected by the hash.
 784  */
 785 static                  uint16_t
 786 siena_filter_tbl_increment(
 787         __in            uint32_t key)
 788 {
 789         return ((uint16_t)(key * 2 - 1));
 790 }
 791
 792 static  __checkReturn   boolean_t
 793 siena_filter_test_used(
 794         __in            siena_filter_tbl_t *sftp,
 795         __in            unsigned int index)
 796 {
 797         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 798         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 799 }
 800
 801 static                  void
 802 siena_filter_set_used(
 803         __in            siena_filter_tbl_t *sftp,
 804         __in            unsigned int index)
 805 {
 806         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 807         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 808         ++sftp->sft_used;
 809 }
 810
 811 static                  void
 812 siena_filter_clear_used(
 813         __in            siena_filter_tbl_t *sftp,
 814         __in            unsigned int index)
 815 {
 816         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 817         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 818
 819         --sftp->sft_used;
 820         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 821 }
 822
 823
 824 static                  siena_filter_tbl_id_t
 825 siena_filter_tbl_id(
 826         __in            siena_filter_type_t type)
 827 {
 828         siena_filter_tbl_id_t tbl_id;
 829
 830         switch (type) {
 831         case EFX_SIENA_FILTER_RX_TCP_FULL:
 832         case EFX_SIENA_FILTER_RX_TCP_WILD:
 833         case EFX_SIENA_FILTER_RX_UDP_FULL:
 834         case EFX_SIENA_FILTER_RX_UDP_WILD:
 835                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 836                 break;
 837
 838         case EFX_SIENA_FILTER_RX_MAC_FULL:
 839         case EFX_SIENA_FILTER_RX_MAC_WILD:
 840                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 841                 break;
 842
 843         case EFX_SIENA_FILTER_TX_TCP_FULL:
 844         case EFX_SIENA_FILTER_TX_TCP_WILD:
 845         case EFX_SIENA_FILTER_TX_UDP_FULL:
 846         case EFX_SIENA_FILTER_TX_UDP_WILD:
 847                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 848                 break;
 849
 850         case EFX_SIENA_FILTER_TX_MAC_FULL:
 851         case EFX_SIENA_FILTER_TX_MAC_WILD:
 852                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 853                 break;
 854
 855         default:
 856                 EFSYS_ASSERT(B_FALSE);
 857                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 858                 break;
 859         }
 860         return (tbl_id);
 861 }
 862
 863 static                  void
 864 siena_filter_reset_search_depth(
 865         __inout         siena_filter_t *sfp,
 866         __in            siena_filter_tbl_id_t tbl_id)
 867 {
 868         switch (tbl_id) {
 869         case EFX_SIENA_FILTER_TBL_RX_IP:
 870                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 871                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 872                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 873                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 874                 break;
 875
 876         case EFX_SIENA_FILTER_TBL_RX_MAC:
 877                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 878                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 879                 break;
 880
 881         case EFX_SIENA_FILTER_TBL_TX_IP:
 882                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 883                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 884                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 885                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 886                 break;
 887
 888         case EFX_SIENA_FILTER_TBL_TX_MAC:
 889                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 890                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 891                 break;
 892
 893         default:
 894                 EFSYS_ASSERT(B_FALSE);
 895                 break;
 896         }
 897 }
 898
 899 static                  void
 900 siena_filter_push_rx_limits(
 901         __in            efx_nic_t *enp)
 902 {
 903         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 904         efx_oword_t oword;
 905
 906         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 907
 908         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 909             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 910             FILTER_CTL_SRCH_FUDGE_FULL);
 911         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 912             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 913             FILTER_CTL_SRCH_FUDGE_WILD);
 914         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 915             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 916             FILTER_CTL_SRCH_FUDGE_FULL);
 917         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 918             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 919             FILTER_CTL_SRCH_FUDGE_WILD);
 920
 921         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 922                 EFX_SET_OWORD_FIELD(oword,
 923                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 924                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 925                     FILTER_CTL_SRCH_FUDGE_FULL);
 926                 EFX_SET_OWORD_FIELD(oword,
 927                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 928                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 929                     FILTER_CTL_SRCH_FUDGE_WILD);
 930         }
 931
 932         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 933 }
 934
 935 static                  void
 936 siena_filter_push_tx_limits(
 937         __in            efx_nic_t *enp)
 938 {
 939         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 940         efx_oword_t oword;
 941
 942         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 943
 944         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 945                 EFX_SET_OWORD_FIELD(oword,
 946                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 947                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 948                     FILTER_CTL_SRCH_FUDGE_FULL);
 949                 EFX_SET_OWORD_FIELD(oword,
 950                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 951                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 952                     FILTER_CTL_SRCH_FUDGE_WILD);
 953                 EFX_SET_OWORD_FIELD(oword,
 954                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 955                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 956                     FILTER_CTL_SRCH_FUDGE_FULL);
 957                 EFX_SET_OWORD_FIELD(oword,
 958                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 959                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 960                     FILTER_CTL_SRCH_FUDGE_WILD);
 961         }
 962
 963         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
 964                 EFX_SET_OWORD_FIELD(
 965                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 966                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
 967                         FILTER_CTL_SRCH_FUDGE_FULL);
 968                 EFX_SET_OWORD_FIELD(
 969                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 970                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
 971                         FILTER_CTL_SRCH_FUDGE_WILD);
 972         }
 973
 974         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 975 }
 976
 977 /* Build a filter entry and return its n-tuple key. */
 978 static  __checkReturn   uint32_t
 979 siena_filter_build(
 980         __out           efx_oword_t *filter,
 981         __in            siena_filter_spec_t *spec)
 982 {
 983         uint32_t dword3;
 984         uint32_t key;
 985         uint8_t  type  = spec->sfs_type;
 986         uint32_t flags = spec->sfs_flags;
 987
 988         switch (siena_filter_tbl_id(type)) {
 989         case EFX_SIENA_FILTER_TBL_RX_IP: {
 990                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
 991                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
 992                 EFX_POPULATE_OWORD_7(*filter,
 993                     FRF_BZ_RSS_EN,
 994                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 995                     FRF_BZ_SCATTER_EN,
 996                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 997                     FRF_AZ_TCP_UDP, is_udp,
 998                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
 999                     EFX_DWORD_2, spec->sfs_dword[2],
1000                     EFX_DWORD_1, spec->sfs_dword[1],
1001                     EFX_DWORD_0, spec->sfs_dword[0]);
1002                 dword3 = is_udp;
1003                 break;
1004         }
1005
1006         case EFX_SIENA_FILTER_TBL_RX_MAC: {
1007                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
1008                 EFX_POPULATE_OWORD_7(*filter,
1009                     FRF_CZ_RMFT_RSS_EN,
1010                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1011                     FRF_CZ_RMFT_SCATTER_EN,
1012                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1013                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
1014                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
1015                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
1016                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
1017                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
1018                 dword3 = is_wild;
1019                 break;
1020         }
1021
1022         case EFX_SIENA_FILTER_TBL_TX_IP: {
1023                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
1024                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
1025                 EFX_POPULATE_OWORD_5(*filter,
1026                     FRF_CZ_TIFT_TCP_UDP, is_udp,
1027                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
1028                     EFX_DWORD_2, spec->sfs_dword[2],
1029                     EFX_DWORD_1, spec->sfs_dword[1],
1030                     EFX_DWORD_0, spec->sfs_dword[0]);
1031                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
1032                 break;
1033         }
1034
1035         case EFX_SIENA_FILTER_TBL_TX_MAC: {
1036                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
1037                 EFX_POPULATE_OWORD_5(*filter,
1038                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
1039                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
1040                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
1041                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
1042                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
1043                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
1044                 break;
1045         }
1046
1047         default:
1048                 EFSYS_ASSERT(B_FALSE);
1049                 EFX_ZERO_OWORD(*filter);
1050                 return (0);
1051         }
1052
1053         key =
1054             spec->sfs_dword[0] ^
1055             spec->sfs_dword[1] ^
1056             spec->sfs_dword[2] ^
1057             dword3;
1058
1059         return (key);
1060 }
1061
1062 static  __checkReturn           efx_rc_t
1063 siena_filter_push_entry(
1064         __inout                 efx_nic_t *enp,
1065         __in                    siena_filter_type_t type,
1066         __in                    int index,
1067         __in                    efx_oword_t *eop)
1068 {
1069         efx_rc_t rc;
1070
1071         switch (type) {
1072         case EFX_SIENA_FILTER_RX_TCP_FULL:
1073         case EFX_SIENA_FILTER_RX_TCP_WILD:
1074         case EFX_SIENA_FILTER_RX_UDP_FULL:
1075         case EFX_SIENA_FILTER_RX_UDP_WILD:
1076                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1077                     eop, B_TRUE);
1078                 break;
1079
1080         case EFX_SIENA_FILTER_RX_MAC_FULL:
1081         case EFX_SIENA_FILTER_RX_MAC_WILD:
1082                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1083                     eop, B_TRUE);
1084                 break;
1085
1086         case EFX_SIENA_FILTER_TX_TCP_FULL:
1087         case EFX_SIENA_FILTER_TX_TCP_WILD:
1088         case EFX_SIENA_FILTER_TX_UDP_FULL:
1089         case EFX_SIENA_FILTER_TX_UDP_WILD:
1090                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1091                     eop, B_TRUE);
1092                 break;
1093
1094         case EFX_SIENA_FILTER_TX_MAC_FULL:
1095         case EFX_SIENA_FILTER_TX_MAC_WILD:
1096                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1097                     eop, B_TRUE);
1098                 break;
1099
1100         default:
1101                 EFSYS_ASSERT(B_FALSE);
1102                 rc = ENOTSUP;
1103                 goto fail1;
1104         }
1105         return (0);
1106
1107 fail1:
1108         return (rc);
1109 }
1110
1111
1112 static  __checkReturn   boolean_t
1113 siena_filter_equal(
1114         __in            const siena_filter_spec_t *left,
1115         __in            const siena_filter_spec_t *right)
1116 {
1117         siena_filter_tbl_id_t tbl_id;
1118
1119         tbl_id = siena_filter_tbl_id(left->sfs_type);
1120
1121
1122         if (left->sfs_type != right->sfs_type)
1123                 return (B_FALSE);
1124
1125         if (memcmp(left->sfs_dword, right->sfs_dword,
1126                 sizeof (left->sfs_dword)))
1127                 return (B_FALSE);
1128
1129         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1130                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1131             left->sfs_dmaq_id != right->sfs_dmaq_id)
1132                 return (B_FALSE);
1133
1134         return (B_TRUE);
1135 }
1136
1137 static  __checkReturn   efx_rc_t
1138 siena_filter_search(
1139         __in            siena_filter_tbl_t *sftp,
1140         __in            siena_filter_spec_t *spec,
1141         __in            uint32_t key,
1142         __in            boolean_t for_insert,
1143         __out           int *filter_index,
1144         __out           unsigned int *depth_required)
1145 {
1146         unsigned int hash, incr, filter_idx, depth;
1147
1148         hash = siena_filter_tbl_hash(key);
1149         incr = siena_filter_tbl_increment(key);
1150
1151         filter_idx = hash & (sftp->sft_size - 1);
1152         depth = 1;
1153
1154         for (;;) {
1155                 /*
1156                  * Return success if entry is used and matches this spec
1157                  * or entry is unused and we are trying to insert.
1158                  */
1159                 if (siena_filter_test_used(sftp, filter_idx) ?
1160                     siena_filter_equal(spec,
1161                     &sftp->sft_spec[filter_idx]) :
1162                     for_insert) {
1163                         *filter_index = filter_idx;
1164                         *depth_required = depth;
1165                         return (0);
1166                 }
1167
1168                 /* Return failure if we reached the maximum search depth */
1169                 if (depth == FILTER_CTL_SRCH_MAX)
1170                         return (for_insert ? EBUSY : ENOENT);
1171
1172                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1173                 ++depth;
1174         }
1175 }
1176
1177 static                  void
1178 siena_filter_clear_entry(
1179         __in            efx_nic_t *enp,
1180         __in            siena_filter_tbl_t *sftp,
1181         __in            int index)
1182 {
1183         efx_oword_t filter;
1184
1185         if (siena_filter_test_used(sftp, index)) {
1186                 siena_filter_clear_used(sftp, index);
1187
1188                 EFX_ZERO_OWORD(filter);
1189                 siena_filter_push_entry(enp,
1190                     sftp->sft_spec[index].sfs_type,
1191                     index, &filter);
1192
1193                 memset(&sftp->sft_spec[index],
1194                     0, sizeof (sftp->sft_spec[0]));
1195         }
1196 }
1197
1198                         void
1199 siena_filter_tbl_clear(
1200         __in            efx_nic_t *enp,
1201         __in            siena_filter_tbl_id_t tbl_id)
1202 {
1203         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1204         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1205         int index;
1206         efsys_lock_state_t state;
1207
1208         EFSYS_LOCK(enp->en_eslp, state);
1209
1210         for (index = 0; index < sftp->sft_size; ++index) {
1211                 siena_filter_clear_entry(enp, sftp, index);
1212         }
1213
1214         if (sftp->sft_used == 0)
1215                 siena_filter_reset_search_depth(sfp, tbl_id);
1216
1217         EFSYS_UNLOCK(enp->en_eslp, state);
1218 }
1219
1220 static  __checkReturn   efx_rc_t
1221 siena_filter_init(
1222         __in            efx_nic_t *enp)
1223 {
1224         siena_filter_t *sfp;
1225         siena_filter_tbl_t *sftp;
1226         int tbl_id;
1227         efx_rc_t rc;
1228
1229         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1230
1231         if (!sfp) {
1232                 rc = ENOMEM;
1233                 goto fail1;
1234         }
1235
1236         enp->en_filter.ef_siena_filter = sfp;
1237
1238         switch (enp->en_family) {
1239         case EFX_FAMILY_SIENA:
1240                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1241                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1242
1243                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1244                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1245
1246                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1247                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1248
1249                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1250                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1251                 break;
1252
1253         default:
1254                 rc = ENOTSUP;
1255                 goto fail2;
1256         }
1257
1258         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1259                 unsigned int bitmap_size;
1260
1261                 sftp = &sfp->sf_tbl[tbl_id];
1262                 if (sftp->sft_size == 0)
1263                         continue;
1264
1265                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1266                     sizeof (uint32_t));
1267                 bitmap_size =
1268                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1269
1270                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1271                 if (!sftp->sft_bitmap) {
1272                         rc = ENOMEM;
1273                         goto fail3;
1274                 }
1275
1276                 EFSYS_KMEM_ALLOC(enp->en_esip,
1277                     sftp->sft_size * sizeof (*sftp->sft_spec),
1278                     sftp->sft_spec);
1279                 if (!sftp->sft_spec) {
1280                         rc = ENOMEM;
1281                         goto fail4;
1282                 }
1283                 memset(sftp->sft_spec, 0,
1284                     sftp->sft_size * sizeof (*sftp->sft_spec));
1285         }
1286
1287         return (0);
1288
1289 fail4:
1290         EFSYS_PROBE(fail4);
1291
1292 fail3:
1293         EFSYS_PROBE(fail3);
1294
1295 fail2:
1296         EFSYS_PROBE(fail2);
1297         siena_filter_fini(enp);
1298
1299 fail1:
1300         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1301         return (rc);
1302 }
1303
1304 static                  void
1305 siena_filter_fini(
1306         __in            efx_nic_t *enp)
1307 {
1308         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1309         siena_filter_tbl_id_t tbl_id;
1310
1311         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1312         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1313
1314         if (sfp == NULL)
1315                 return;
1316
1317         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1318                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1319                 unsigned int bitmap_size;
1320
1321                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1322                     sizeof (uint32_t));
1323                 bitmap_size =
1324                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1325
1326                 if (sftp->sft_bitmap != NULL) {
1327                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1328                             sftp->sft_bitmap);
1329                         sftp->sft_bitmap = NULL;
1330                 }
1331
1332                 if (sftp->sft_spec != NULL) {
1333                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1334                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1335                         sftp->sft_spec = NULL;
1336                 }
1337         }
1338
1339         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1340             enp->en_filter.ef_siena_filter);
1341 }
1342
1343 /* Restore filter state after a reset */
1344 static  __checkReturn   efx_rc_t
1345 siena_filter_restore(
1346         __in            efx_nic_t *enp)
1347 {
1348         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1349         siena_filter_tbl_id_t tbl_id;
1350         siena_filter_tbl_t *sftp;
1351         siena_filter_spec_t *spec;
1352         efx_oword_t filter;
1353         int filter_idx;
1354         efsys_lock_state_t state;
1355         uint32_t key;
1356         efx_rc_t rc;
1357
1358         EFSYS_LOCK(enp->en_eslp, state);
1359
1360         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1361                 sftp = &sfp->sf_tbl[tbl_id];
1362                 for (filter_idx = 0;
1363                         filter_idx < sftp->sft_size;
1364                         filter_idx++) {
1365                         if (!siena_filter_test_used(sftp, filter_idx))
1366                                 continue;
1367
1368                         spec = &sftp->sft_spec[filter_idx];
1369                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1370                                 rc = EINVAL;
1371                                 goto fail1;
1372                         }
1373                         if ((rc = siena_filter_push_entry(enp,
1374                                     spec->sfs_type, filter_idx, &filter)) != 0)
1375                                 goto fail2;
1376                 }
1377         }
1378
1379         siena_filter_push_rx_limits(enp);
1380         siena_filter_push_tx_limits(enp);
1381
1382         EFSYS_UNLOCK(enp->en_eslp, state);
1383
1384         return (0);
1385
1386 fail2:
1387         EFSYS_PROBE(fail2);
1388
1389 fail1:
1390         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1391
1392         EFSYS_UNLOCK(enp->en_eslp, state);
1393
1394         return (rc);
1395 }
1396
1397 static   __checkReturn  efx_rc_t
1398 siena_filter_add(
1399         __in            efx_nic_t *enp,
1400         __inout         efx_filter_spec_t *spec,
1401         __in            boolean_t may_replace)
1402 {
1403         efx_rc_t rc;
1404         siena_filter_spec_t sf_spec;
1405         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1406         siena_filter_tbl_id_t tbl_id;
1407         siena_filter_tbl_t *sftp;
1408         siena_filter_spec_t *saved_sf_spec;
1409         efx_oword_t filter;
1410         int filter_idx;
1411         unsigned int depth;
1412         efsys_lock_state_t state;
1413         uint32_t key;
1414
1415
1416         EFSYS_ASSERT3P(spec, !=, NULL);
1417
1418         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1419                 goto fail1;
1420
1421         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1422         sftp = &sfp->sf_tbl[tbl_id];
1423
1424         if (sftp->sft_size == 0) {
1425                 rc = EINVAL;
1426                 goto fail2;
1427         }
1428
1429         key = siena_filter_build(&filter, &sf_spec);
1430
1431         EFSYS_LOCK(enp->en_eslp, state);
1432
1433         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1434             &filter_idx, &depth);
1435         if (rc != 0)
1436                 goto fail3;
1437
1438         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1439         saved_sf_spec = &sftp->sft_spec[filter_idx];
1440
1441         if (siena_filter_test_used(sftp, filter_idx)) {
1442                 if (may_replace == B_FALSE) {
1443                         rc = EEXIST;
1444                         goto fail4;
1445                 }
1446         }
1447         siena_filter_set_used(sftp, filter_idx);
1448         *saved_sf_spec = sf_spec;
1449
1450         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1451                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1452                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1453                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1454                         siena_filter_push_tx_limits(enp);
1455                 else
1456                         siena_filter_push_rx_limits(enp);
1457         }
1458
1459         siena_filter_push_entry(enp, sf_spec.sfs_type,
1460             filter_idx, &filter);
1461
1462         EFSYS_UNLOCK(enp->en_eslp, state);
1463         return (0);
1464
1465 fail4:
1466         EFSYS_PROBE(fail4);
1467
1468 fail3:
1469         EFSYS_UNLOCK(enp->en_eslp, state);
1470         EFSYS_PROBE(fail3);
1471
1472 fail2:
1473         EFSYS_PROBE(fail2);
1474
1475 fail1:
1476         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1477         return (rc);
1478 }
1479
1480 static   __checkReturn  efx_rc_t
1481 siena_filter_delete(
1482         __in            efx_nic_t *enp,
1483         __inout         efx_filter_spec_t *spec)
1484 {
1485         efx_rc_t rc;
1486         siena_filter_spec_t sf_spec;
1487         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1488         siena_filter_tbl_id_t tbl_id;
1489         siena_filter_tbl_t *sftp;
1490         efx_oword_t filter;
1491         int filter_idx;
1492         unsigned int depth;
1493         efsys_lock_state_t state;
1494         uint32_t key;
1495
1496         EFSYS_ASSERT3P(spec, !=, NULL);
1497
1498         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1499                 goto fail1;
1500
1501         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1502         sftp = &sfp->sf_tbl[tbl_id];
1503
1504         key = siena_filter_build(&filter, &sf_spec);
1505
1506         EFSYS_LOCK(enp->en_eslp, state);
1507
1508         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1509             &filter_idx, &depth);
1510         if (rc != 0)
1511                 goto fail2;
1512
1513         siena_filter_clear_entry(enp, sftp, filter_idx);
1514         if (sftp->sft_used == 0)
1515                 siena_filter_reset_search_depth(sfp, tbl_id);
1516
1517         EFSYS_UNLOCK(enp->en_eslp, state);
1518         return (0);
1519
1520 fail2:
1521         EFSYS_UNLOCK(enp->en_eslp, state);
1522         EFSYS_PROBE(fail2);
1523
1524 fail1:
1525         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1526         return (rc);
1527 }
1528
1529 #define SIENA_MAX_SUPPORTED_MATCHES 4
1530
1531 static  __checkReturn   efx_rc_t
1532 siena_filter_supported_filters(
1533         __in                            efx_nic_t *enp,
1534         __out_ecount(buffer_length)     uint32_t *buffer,
1535         __in                            size_t buffer_length,
1536         __out                           size_t *list_lengthp)
1537 {
1538         uint32_t index = 0;
1539         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1540         size_t list_length;
1541         efx_rc_t rc;
1542
1543         rx_matches[index++] =
1544             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1545             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1546             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1547
1548         rx_matches[index++] =
1549             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1550             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1551
1552         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1553                 rx_matches[index++] =
1554                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1555
1556                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1557         }
1558
1559         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1560         list_length = index;
1561
1562         *list_lengthp = list_length;
1563
1564         if (buffer_length < list_length) {
1565                 rc = ENOSPC;
1566                 goto fail1;
1567         }
1568
1569         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1570
1571         return (0);
1572
1573 fail1:
1574         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1575
1576         return (rc);
1577 }
1578
1579 #undef MAX_SUPPORTED
1580
1581 #endif /* EFSYS_OPT_SIENA */
1582
1583 #endif /* EFSYS_OPT_FILTER */