sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "efx.h"
  35 #include "efx_impl.h"
  36
  37
  38 #if EFSYS_OPT_FILTER
  39
  40 #if EFSYS_OPT_SIENA
  41
  42 static  __checkReturn   efx_rc_t
  43 siena_filter_init(
  44         __in            efx_nic_t *enp);
  45
  46 static                  void
  47 siena_filter_fini(
  48         __in            efx_nic_t *enp);
  49
  50 static  __checkReturn   efx_rc_t
  51 siena_filter_restore(
  52         __in            efx_nic_t *enp);
  53
  54 static  __checkReturn   efx_rc_t
  55 siena_filter_add(
  56         __in            efx_nic_t *enp,
  57         __inout         efx_filter_spec_t *spec,
  58         __in            boolean_t may_replace);
  59
  60 static  __checkReturn   efx_rc_t
  61 siena_filter_delete(
  62         __in            efx_nic_t *enp,
  63         __inout         efx_filter_spec_t *spec);
  64
  65 static  __checkReturn   efx_rc_t
  66 siena_filter_supported_filters(
  67         __in                            efx_nic_t *enp,
  68         __out_ecount(buffer_length)     uint32_t *buffer,
  69         __in                            size_t buffer_length,
  70         __out                           size_t *list_lengthp);
  71
  72 #endif /* EFSYS_OPT_SIENA */
  73
  74 #if EFSYS_OPT_SIENA
  75 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  76         siena_filter_init,              /* efo_init */
  77         siena_filter_fini,              /* efo_fini */
  78         siena_filter_restore,           /* efo_restore */
  79         siena_filter_add,               /* efo_add */
  80         siena_filter_delete,            /* efo_delete */
  81         siena_filter_supported_filters, /* efo_supported_filters */
  82         NULL,                           /* efo_reconfigure */
  83 };
  84 #endif /* EFSYS_OPT_SIENA */
  85
  86 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  87 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  88         ef10_filter_init,               /* efo_init */
  89         ef10_filter_fini,               /* efo_fini */
  90         ef10_filter_restore,            /* efo_restore */
  91         ef10_filter_add,                /* efo_add */
  92         ef10_filter_delete,             /* efo_delete */
  93         ef10_filter_supported_filters,  /* efo_supported_filters */
  94         ef10_filter_reconfigure,        /* efo_reconfigure */
  95 };
  96 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
  97
  98         __checkReturn   efx_rc_t
  99 efx_filter_insert(
 100         __in            efx_nic_t *enp,
 101         __inout         efx_filter_spec_t *spec)
 102 {
 103         const efx_filter_ops_t *efop = enp->en_efop;
 104
 105         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 106         EFSYS_ASSERT3P(spec, !=, NULL);
 107         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 108
 109         return (efop->efo_add(enp, spec, B_FALSE));
 110 }
 111
 112         __checkReturn   efx_rc_t
 113 efx_filter_remove(
 114         __in            efx_nic_t *enp,
 115         __inout         efx_filter_spec_t *spec)
 116 {
 117         const efx_filter_ops_t *efop = enp->en_efop;
 118
 119         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 120         EFSYS_ASSERT3P(spec, !=, NULL);
 121         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 122
 123 #if EFSYS_OPT_RX_SCALE
 124         spec->efs_rss_context = enp->en_rss_context;
 125 #endif
 126
 127         return (efop->efo_delete(enp, spec));
 128 }
 129
 130         __checkReturn   efx_rc_t
 131 efx_filter_restore(
 132         __in            efx_nic_t *enp)
 133 {
 134         efx_rc_t rc;
 135
 136         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 137
 138         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 139                 goto fail1;
 140
 141         return (0);
 142
 143 fail1:
 144         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 145
 146         return (rc);
 147 }
 148
 149         __checkReturn   efx_rc_t
 150 efx_filter_init(
 151         __in            efx_nic_t *enp)
 152 {
 153         const efx_filter_ops_t *efop;
 154         efx_rc_t rc;
 155
 156         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 157         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 158         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 159
 160         switch (enp->en_family) {
 161 #if EFSYS_OPT_SIENA
 162         case EFX_FAMILY_SIENA:
 163                 efop = &__efx_filter_siena_ops;
 164                 break;
 165 #endif /* EFSYS_OPT_SIENA */
 166
 167 #if EFSYS_OPT_HUNTINGTON
 168         case EFX_FAMILY_HUNTINGTON:
 169                 efop = &__efx_filter_ef10_ops;
 170                 break;
 171 #endif /* EFSYS_OPT_HUNTINGTON */
 172
 173 #if EFSYS_OPT_MEDFORD
 174         case EFX_FAMILY_MEDFORD:
 175                 efop = &__efx_filter_ef10_ops;
 176                 break;
 177 #endif /* EFSYS_OPT_MEDFORD */
 178
 179         default:
 180                 EFSYS_ASSERT(0);
 181                 rc = ENOTSUP;
 182                 goto fail1;
 183         }
 184
 185         if ((rc = efop->efo_init(enp)) != 0)
 186                 goto fail2;
 187
 188         enp->en_efop = efop;
 189         enp->en_mod_flags |= EFX_MOD_FILTER;
 190         return (0);
 191
 192 fail2:
 193         EFSYS_PROBE(fail2);
 194 fail1:
 195         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 196
 197         enp->en_efop = NULL;
 198         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 199         return (rc);
 200 }
 201
 202                         void
 203 efx_filter_fini(
 204         __in            efx_nic_t *enp)
 205 {
 206         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 207         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 208         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 209
 210         enp->en_efop->efo_fini(enp);
 211
 212         enp->en_efop = NULL;
 213         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 214 }
 215
 216 /*
 217  * Query the possible combinations of match flags which can be filtered on.
 218  * These are returned as a list, of which each 32 bit element is a bitmask
 219  * formed of EFX_FILTER_MATCH flags.
 220  *
 221  * The combinations are ordered in priority from highest to lowest.
 222  *
 223  * If the provided buffer is too short to hold the list, the call with fail with
 224  * ENOSPC and *list_lengthp will be set to the buffer length required.
 225  */
 226         __checkReturn   efx_rc_t
 227 efx_filter_supported_filters(
 228         __in                            efx_nic_t *enp,
 229         __out_ecount(buffer_length)     uint32_t *buffer,
 230         __in                            size_t buffer_length,
 231         __out                           size_t *list_lengthp)
 232 {
 233         efx_rc_t rc;
 234
 235         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 236         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 237         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 238         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 239
 240         if (buffer == NULL) {
 241                 rc = EINVAL;
 242                 goto fail1;
 243         }
 244
 245         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 246                                                     list_lengthp);
 247         if (rc != 0)
 248                 goto fail2;
 249
 250         return (0);
 251
 252 fail2:
 253         EFSYS_PROBE(fail2);
 254 fail1:
 255         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 256
 257         return (rc);
 258 }
 259
 260         __checkReturn   efx_rc_t
 261 efx_filter_reconfigure(
 262         __in                            efx_nic_t *enp,
 263         __in_ecount(6)                  uint8_t const *mac_addr,
 264         __in                            boolean_t all_unicst,
 265         __in                            boolean_t mulcst,
 266         __in                            boolean_t all_mulcst,
 267         __in                            boolean_t brdcst,
 268         __in_ecount(6*count)            uint8_t const *addrs,
 269         __in                            uint32_t count)
 270 {
 271         efx_rc_t rc;
 272
 273         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 274         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 275         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 276
 277         if (enp->en_efop->efo_reconfigure != NULL) {
 278                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 279                                                         all_unicst, mulcst,
 280                                                         all_mulcst, brdcst,
 281                                                         addrs, count)) != 0)
 282                         goto fail1;
 283         }
 284
 285         return (0);
 286
 287 fail1:
 288         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 289
 290         return (rc);
 291 }
 292
 293                 void
 294 efx_filter_spec_init_rx(
 295         __out           efx_filter_spec_t *spec,
 296         __in            efx_filter_priority_t priority,
 297         __in            efx_filter_flags_t flags,
 298         __in            efx_rxq_t *erp)
 299 {
 300         EFSYS_ASSERT3P(spec, !=, NULL);
 301         EFSYS_ASSERT3P(erp, !=, NULL);
 302         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 303                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 304
 305         memset(spec, 0, sizeof (*spec));
 306         spec->efs_priority = priority;
 307         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 308         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 309         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 310 }
 311
 312                 void
 313 efx_filter_spec_init_tx(
 314         __out           efx_filter_spec_t *spec,
 315         __in            efx_txq_t *etp)
 316 {
 317         EFSYS_ASSERT3P(spec, !=, NULL);
 318         EFSYS_ASSERT3P(etp, !=, NULL);
 319
 320         memset(spec, 0, sizeof (*spec));
 321         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 322         spec->efs_flags = EFX_FILTER_FLAG_TX;
 323         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 324 }
 325
 326
 327 /*
 328  *  Specify IPv4 host, transport protocol and port in a filter specification
 329  */
 330 __checkReturn           efx_rc_t
 331 efx_filter_spec_set_ipv4_local(
 332         __inout         efx_filter_spec_t *spec,
 333         __in            uint8_t proto,
 334         __in            uint32_t host,
 335         __in            uint16_t port)
 336 {
 337         EFSYS_ASSERT3P(spec, !=, NULL);
 338
 339         spec->efs_match_flags |=
 340                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 341                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 342         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 343         spec->efs_ip_proto = proto;
 344         spec->efs_loc_host.eo_u32[0] = host;
 345         spec->efs_loc_port = port;
 346         return (0);
 347 }
 348
 349 /*
 350  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 351  */
 352 __checkReturn           efx_rc_t
 353 efx_filter_spec_set_ipv4_full(
 354         __inout         efx_filter_spec_t *spec,
 355         __in            uint8_t proto,
 356         __in            uint32_t lhost,
 357         __in            uint16_t lport,
 358         __in            uint32_t rhost,
 359         __in            uint16_t rport)
 360 {
 361         EFSYS_ASSERT3P(spec, !=, NULL);
 362
 363         spec->efs_match_flags |=
 364                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 365                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 366                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 367         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 368         spec->efs_ip_proto = proto;
 369         spec->efs_loc_host.eo_u32[0] = lhost;
 370         spec->efs_loc_port = lport;
 371         spec->efs_rem_host.eo_u32[0] = rhost;
 372         spec->efs_rem_port = rport;
 373         return (0);
 374 }
 375
 376 /*
 377  * Specify local Ethernet address and/or VID in filter specification
 378  */
 379 __checkReturn           efx_rc_t
 380 efx_filter_spec_set_eth_local(
 381         __inout         efx_filter_spec_t *spec,
 382         __in            uint16_t vid,
 383         __in            const uint8_t *addr)
 384 {
 385         EFSYS_ASSERT3P(spec, !=, NULL);
 386         EFSYS_ASSERT3P(addr, !=, NULL);
 387
 388         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 389                 return (EINVAL);
 390
 391         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 392                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 393                 spec->efs_outer_vid = vid;
 394         }
 395         if (addr != NULL) {
 396                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 397                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 398         }
 399         return (0);
 400 }
 401
 402                         void
 403 efx_filter_spec_set_ether_type(
 404         __inout         efx_filter_spec_t *spec,
 405         __in            uint16_t ether_type)
 406 {
 407         EFSYS_ASSERT3P(spec, !=, NULL);
 408
 409         spec->efs_ether_type = ether_type;
 410         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 411 }
 412
 413 /*
 414  * Specify matching otherwise-unmatched unicast in a filter specification
 415  */
 416 __checkReturn           efx_rc_t
 417 efx_filter_spec_set_uc_def(
 418         __inout         efx_filter_spec_t *spec)
 419 {
 420         EFSYS_ASSERT3P(spec, !=, NULL);
 421
 422         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 423         return (0);
 424 }
 425
 426 /*
 427  * Specify matching otherwise-unmatched multicast in a filter specification
 428  */
 429 __checkReturn           efx_rc_t
 430 efx_filter_spec_set_mc_def(
 431         __inout         efx_filter_spec_t *spec)
 432 {
 433         EFSYS_ASSERT3P(spec, !=, NULL);
 434
 435         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 436         return (0);
 437 }
 438
 439
 440 __checkReturn           efx_rc_t
 441 efx_filter_spec_set_encap_type(
 442         __inout         efx_filter_spec_t *spec,
 443         __in            efx_tunnel_protocol_t encap_type,
 444         __in            efx_filter_inner_frame_match_t inner_frame_match)
 445 {
 446         uint32_t match_flags = 0;
 447         uint8_t ip_proto;
 448         efx_rc_t rc;
 449
 450         EFSYS_ASSERT3P(spec, !=, NULL);
 451
 452         switch (encap_type) {
 453         case EFX_TUNNEL_PROTOCOL_VXLAN:
 454         case EFX_TUNNEL_PROTOCOL_GENEVE:
 455                 ip_proto = EFX_IPPROTO_UDP;
 456                 break;
 457         case EFX_TUNNEL_PROTOCOL_NVGRE:
 458                 ip_proto = EFX_IPPROTO_GRE;
 459                 break;
 460         default:
 461                 EFSYS_ASSERT(0);
 462                 rc = EINVAL;
 463                 goto fail1;
 464         }
 465
 466         switch (inner_frame_match) {
 467         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 468                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 469                 break;
 470         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 471                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 472                 break;
 473         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 474                 /* This is for when specific inner frames are to be matched. */
 475                 break;
 476         default:
 477                 EFSYS_ASSERT(0);
 478                 rc = EINVAL;
 479                 goto fail2;
 480         }
 481
 482         spec->efs_encap_type = encap_type;
 483         spec->efs_ip_proto = ip_proto;
 484         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 485
 486         return (0);
 487
 488 fail2:
 489         EFSYS_PROBE(fail2);
 490 fail1:
 491         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 492
 493         return (rc);
 494 }
 495
 496
 497
 498 #if EFSYS_OPT_SIENA
 499
 500 /*
 501  * "Fudge factors" - difference between programmed value and actual depth.
 502  * Due to pipelined implementation we need to program H/W with a value that
 503  * is larger than the hop limit we want.
 504  */
 505 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 506 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 507
 508 /*
 509  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 510  * We also need to avoid infinite loops in efx_filter_search() when the
 511  * table is full.
 512  */
 513 #define FILTER_CTL_SRCH_MAX 200
 514
 515 static  __checkReturn   efx_rc_t
 516 siena_filter_spec_from_gen_spec(
 517         __out           siena_filter_spec_t *sf_spec,
 518         __in            efx_filter_spec_t *gen_spec)
 519 {
 520         efx_rc_t rc;
 521         boolean_t is_full = B_FALSE;
 522
 523         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 524                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 525         else
 526                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 527
 528         /* Falconsiena only has one RSS context */
 529         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 530             gen_spec->efs_rss_context != 0) {
 531                 rc = EINVAL;
 532                 goto fail1;
 533         }
 534
 535         sf_spec->sfs_flags = gen_spec->efs_flags;
 536         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 537
 538         switch (gen_spec->efs_match_flags) {
 539         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 540             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 541             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 542                 is_full = B_TRUE;
 543                 /* Fall through */
 544         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 545             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 546                 uint32_t rhost, host1, host2;
 547                 uint16_t rport, port1, port2;
 548
 549                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 550                         rc = ENOTSUP;
 551                         goto fail2;
 552                 }
 553                 if (gen_spec->efs_loc_port == 0 ||
 554                     (is_full && gen_spec->efs_rem_port == 0)) {
 555                         rc = EINVAL;
 556                         goto fail3;
 557                 }
 558                 switch (gen_spec->efs_ip_proto) {
 559                 case EFX_IPPROTO_TCP:
 560                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 561                                 sf_spec->sfs_type = (is_full ?
 562                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 563                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 564                         } else {
 565                                 sf_spec->sfs_type = (is_full ?
 566                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 567                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 568                         }
 569                         break;
 570                 case EFX_IPPROTO_UDP:
 571                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 572                                 sf_spec->sfs_type = (is_full ?
 573                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 574                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 575                         } else {
 576                                 sf_spec->sfs_type = (is_full ?
 577                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 578                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 579                         }
 580                         break;
 581                 default:
 582                         rc = ENOTSUP;
 583                         goto fail4;
 584                 }
 585                 /*
 586                  * The filter is constructed in terms of source and destination,
 587                  * with the odd wrinkle that the ports are swapped in a UDP
 588                  * wildcard filter. We need to convert from local and remote
 589                  * addresses (zero for a wildcard).
 590                  */
 591                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 592                 rport = is_full ? gen_spec->efs_rem_port : 0;
 593                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 594                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 595                         host2 = rhost;
 596                 } else {
 597                         host1 = rhost;
 598                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 599                 }
 600                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 601                         if (sf_spec->sfs_type ==
 602                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 603                                 port1 = rport;
 604                                 port2 = gen_spec->efs_loc_port;
 605                         } else {
 606                                 port1 = gen_spec->efs_loc_port;
 607                                 port2 = rport;
 608                         }
 609                 } else {
 610                         if (sf_spec->sfs_type ==
 611                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 612                                 port1 = gen_spec->efs_loc_port;
 613                                 port2 = rport;
 614                         } else {
 615                                 port1 = rport;
 616                                 port2 = gen_spec->efs_loc_port;
 617                         }
 618                 }
 619                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 620                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 621                 sf_spec->sfs_dword[2] = host2;
 622                 break;
 623         }
 624
 625         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 626                 is_full = B_TRUE;
 627                 /* Fall through */
 628         case EFX_FILTER_MATCH_LOC_MAC:
 629                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 630                         sf_spec->sfs_type = (is_full ?
 631                             EFX_SIENA_FILTER_TX_MAC_FULL :
 632                             EFX_SIENA_FILTER_TX_MAC_WILD);
 633                 } else {
 634                         sf_spec->sfs_type = (is_full ?
 635                             EFX_SIENA_FILTER_RX_MAC_FULL :
 636                             EFX_SIENA_FILTER_RX_MAC_WILD);
 637                 }
 638                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 639                 sf_spec->sfs_dword[1] =
 640                     gen_spec->efs_loc_mac[2] << 24 |
 641                     gen_spec->efs_loc_mac[3] << 16 |
 642                     gen_spec->efs_loc_mac[4] <<  8 |
 643                     gen_spec->efs_loc_mac[5];
 644                 sf_spec->sfs_dword[2] =
 645                     gen_spec->efs_loc_mac[0] << 8 |
 646                     gen_spec->efs_loc_mac[1];
 647                 break;
 648
 649         default:
 650                 EFSYS_ASSERT(B_FALSE);
 651                 rc = ENOTSUP;
 652                 goto fail5;
 653         }
 654
 655         return (0);
 656
 657 fail5:
 658         EFSYS_PROBE(fail5);
 659 fail4:
 660         EFSYS_PROBE(fail4);
 661 fail3:
 662         EFSYS_PROBE(fail3);
 663 fail2:
 664         EFSYS_PROBE(fail2);
 665 fail1:
 666         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 667
 668         return (rc);
 669 }
 670
 671 /*
 672  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 673  * key derived from the n-tuple.
 674  */
 675 static                  uint16_t
 676 siena_filter_tbl_hash(
 677         __in            uint32_t key)
 678 {
 679         uint16_t tmp;
 680
 681         /* First 16 rounds */
 682         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 683         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 684         tmp = tmp ^ tmp >> 9;
 685
 686         /* Last 16 rounds */
 687         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 688         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 689         tmp = tmp ^ tmp >> 9;
 690
 691         return (tmp);
 692 }
 693
 694 /*
 695  * To allow for hash collisions, filter search continues at these
 696  * increments from the first possible entry selected by the hash.
 697  */
 698 static                  uint16_t
 699 siena_filter_tbl_increment(
 700         __in            uint32_t key)
 701 {
 702         return ((uint16_t)(key * 2 - 1));
 703 }
 704
 705 static  __checkReturn   boolean_t
 706 siena_filter_test_used(
 707         __in            siena_filter_tbl_t *sftp,
 708         __in            unsigned int index)
 709 {
 710         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 711         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 712 }
 713
 714 static                  void
 715 siena_filter_set_used(
 716         __in            siena_filter_tbl_t *sftp,
 717         __in            unsigned int index)
 718 {
 719         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 720         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 721         ++sftp->sft_used;
 722 }
 723
 724 static                  void
 725 siena_filter_clear_used(
 726         __in            siena_filter_tbl_t *sftp,
 727         __in            unsigned int index)
 728 {
 729         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 730         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 731
 732         --sftp->sft_used;
 733         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 734 }
 735
 736
 737 static                  siena_filter_tbl_id_t
 738 siena_filter_tbl_id(
 739         __in            siena_filter_type_t type)
 740 {
 741         siena_filter_tbl_id_t tbl_id;
 742
 743         switch (type) {
 744         case EFX_SIENA_FILTER_RX_TCP_FULL:
 745         case EFX_SIENA_FILTER_RX_TCP_WILD:
 746         case EFX_SIENA_FILTER_RX_UDP_FULL:
 747         case EFX_SIENA_FILTER_RX_UDP_WILD:
 748                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 749                 break;
 750
 751         case EFX_SIENA_FILTER_RX_MAC_FULL:
 752         case EFX_SIENA_FILTER_RX_MAC_WILD:
 753                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 754                 break;
 755
 756         case EFX_SIENA_FILTER_TX_TCP_FULL:
 757         case EFX_SIENA_FILTER_TX_TCP_WILD:
 758         case EFX_SIENA_FILTER_TX_UDP_FULL:
 759         case EFX_SIENA_FILTER_TX_UDP_WILD:
 760                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 761                 break;
 762
 763         case EFX_SIENA_FILTER_TX_MAC_FULL:
 764         case EFX_SIENA_FILTER_TX_MAC_WILD:
 765                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 766                 break;
 767
 768         default:
 769                 EFSYS_ASSERT(B_FALSE);
 770                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 771                 break;
 772         }
 773         return (tbl_id);
 774 }
 775
 776 static                  void
 777 siena_filter_reset_search_depth(
 778         __inout         siena_filter_t *sfp,
 779         __in            siena_filter_tbl_id_t tbl_id)
 780 {
 781         switch (tbl_id) {
 782         case EFX_SIENA_FILTER_TBL_RX_IP:
 783                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 784                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 785                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 786                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 787                 break;
 788
 789         case EFX_SIENA_FILTER_TBL_RX_MAC:
 790                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 791                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 792                 break;
 793
 794         case EFX_SIENA_FILTER_TBL_TX_IP:
 795                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 796                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 797                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 798                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 799                 break;
 800
 801         case EFX_SIENA_FILTER_TBL_TX_MAC:
 802                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 803                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 804                 break;
 805
 806         default:
 807                 EFSYS_ASSERT(B_FALSE);
 808                 break;
 809         }
 810 }
 811
 812 static                  void
 813 siena_filter_push_rx_limits(
 814         __in            efx_nic_t *enp)
 815 {
 816         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 817         efx_oword_t oword;
 818
 819         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 820
 821         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 822             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 823             FILTER_CTL_SRCH_FUDGE_FULL);
 824         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 825             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 826             FILTER_CTL_SRCH_FUDGE_WILD);
 827         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 828             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 829             FILTER_CTL_SRCH_FUDGE_FULL);
 830         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 831             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 832             FILTER_CTL_SRCH_FUDGE_WILD);
 833
 834         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 835                 EFX_SET_OWORD_FIELD(oword,
 836                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 837                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 838                     FILTER_CTL_SRCH_FUDGE_FULL);
 839                 EFX_SET_OWORD_FIELD(oword,
 840                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 841                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 842                     FILTER_CTL_SRCH_FUDGE_WILD);
 843         }
 844
 845         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 846 }
 847
 848 static                  void
 849 siena_filter_push_tx_limits(
 850         __in            efx_nic_t *enp)
 851 {
 852         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 853         efx_oword_t oword;
 854
 855         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 856
 857         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 858                 EFX_SET_OWORD_FIELD(oword,
 859                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 860                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 861                     FILTER_CTL_SRCH_FUDGE_FULL);
 862                 EFX_SET_OWORD_FIELD(oword,
 863                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 864                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 865                     FILTER_CTL_SRCH_FUDGE_WILD);
 866                 EFX_SET_OWORD_FIELD(oword,
 867                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 868                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 869                     FILTER_CTL_SRCH_FUDGE_FULL);
 870                 EFX_SET_OWORD_FIELD(oword,
 871                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 872                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 873                     FILTER_CTL_SRCH_FUDGE_WILD);
 874         }
 875
 876         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
 877                 EFX_SET_OWORD_FIELD(
 878                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 879                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
 880                         FILTER_CTL_SRCH_FUDGE_FULL);
 881                 EFX_SET_OWORD_FIELD(
 882                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 883                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
 884                         FILTER_CTL_SRCH_FUDGE_WILD);
 885         }
 886
 887         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 888 }
 889
 890 /* Build a filter entry and return its n-tuple key. */
 891 static  __checkReturn   uint32_t
 892 siena_filter_build(
 893         __out           efx_oword_t *filter,
 894         __in            siena_filter_spec_t *spec)
 895 {
 896         uint32_t dword3;
 897         uint32_t key;
 898         uint8_t  type  = spec->sfs_type;
 899         uint32_t flags = spec->sfs_flags;
 900
 901         switch (siena_filter_tbl_id(type)) {
 902         case EFX_SIENA_FILTER_TBL_RX_IP: {
 903                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
 904                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
 905                 EFX_POPULATE_OWORD_7(*filter,
 906                     FRF_BZ_RSS_EN,
 907                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 908                     FRF_BZ_SCATTER_EN,
 909                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 910                     FRF_AZ_TCP_UDP, is_udp,
 911                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
 912                     EFX_DWORD_2, spec->sfs_dword[2],
 913                     EFX_DWORD_1, spec->sfs_dword[1],
 914                     EFX_DWORD_0, spec->sfs_dword[0]);
 915                 dword3 = is_udp;
 916                 break;
 917         }
 918
 919         case EFX_SIENA_FILTER_TBL_RX_MAC: {
 920                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
 921                 EFX_POPULATE_OWORD_7(*filter,
 922                     FRF_CZ_RMFT_RSS_EN,
 923                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 924                     FRF_CZ_RMFT_SCATTER_EN,
 925                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 926                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
 927                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 928                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
 929                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
 930                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
 931                 dword3 = is_wild;
 932                 break;
 933         }
 934
 935         case EFX_SIENA_FILTER_TBL_TX_IP: {
 936                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
 937                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
 938                 EFX_POPULATE_OWORD_5(*filter,
 939                     FRF_CZ_TIFT_TCP_UDP, is_udp,
 940                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
 941                     EFX_DWORD_2, spec->sfs_dword[2],
 942                     EFX_DWORD_1, spec->sfs_dword[1],
 943                     EFX_DWORD_0, spec->sfs_dword[0]);
 944                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
 945                 break;
 946         }
 947
 948         case EFX_SIENA_FILTER_TBL_TX_MAC: {
 949                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
 950                 EFX_POPULATE_OWORD_5(*filter,
 951                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
 952                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 953                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
 954                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
 955                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
 956                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
 957                 break;
 958         }
 959
 960         default:
 961                 EFSYS_ASSERT(B_FALSE);
 962                 return (0);
 963         }
 964
 965         key =
 966             spec->sfs_dword[0] ^
 967             spec->sfs_dword[1] ^
 968             spec->sfs_dword[2] ^
 969             dword3;
 970
 971         return (key);
 972 }
 973
 974 static  __checkReturn           efx_rc_t
 975 siena_filter_push_entry(
 976         __inout                 efx_nic_t *enp,
 977         __in                    siena_filter_type_t type,
 978         __in                    int index,
 979         __in                    efx_oword_t *eop)
 980 {
 981         efx_rc_t rc;
 982
 983         switch (type) {
 984         case EFX_SIENA_FILTER_RX_TCP_FULL:
 985         case EFX_SIENA_FILTER_RX_TCP_WILD:
 986         case EFX_SIENA_FILTER_RX_UDP_FULL:
 987         case EFX_SIENA_FILTER_RX_UDP_WILD:
 988                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
 989                     eop, B_TRUE);
 990                 break;
 991
 992         case EFX_SIENA_FILTER_RX_MAC_FULL:
 993         case EFX_SIENA_FILTER_RX_MAC_WILD:
 994                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
 995                     eop, B_TRUE);
 996                 break;
 997
 998         case EFX_SIENA_FILTER_TX_TCP_FULL:
 999         case EFX_SIENA_FILTER_TX_TCP_WILD:
1000         case EFX_SIENA_FILTER_TX_UDP_FULL:
1001         case EFX_SIENA_FILTER_TX_UDP_WILD:
1002                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1003                     eop, B_TRUE);
1004                 break;
1005
1006         case EFX_SIENA_FILTER_TX_MAC_FULL:
1007         case EFX_SIENA_FILTER_TX_MAC_WILD:
1008                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1009                     eop, B_TRUE);
1010                 break;
1011
1012         default:
1013                 EFSYS_ASSERT(B_FALSE);
1014                 rc = ENOTSUP;
1015                 goto fail1;
1016         }
1017         return (0);
1018
1019 fail1:
1020         return (rc);
1021 }
1022
1023
1024 static  __checkReturn   boolean_t
1025 siena_filter_equal(
1026         __in            const siena_filter_spec_t *left,
1027         __in            const siena_filter_spec_t *right)
1028 {
1029         siena_filter_tbl_id_t tbl_id;
1030
1031         tbl_id = siena_filter_tbl_id(left->sfs_type);
1032
1033
1034         if (left->sfs_type != right->sfs_type)
1035                 return (B_FALSE);
1036
1037         if (memcmp(left->sfs_dword, right->sfs_dword,
1038                 sizeof (left->sfs_dword)))
1039                 return (B_FALSE);
1040
1041         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1042                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1043             left->sfs_dmaq_id != right->sfs_dmaq_id)
1044                 return (B_FALSE);
1045
1046         return (B_TRUE);
1047 }
1048
1049 static  __checkReturn   efx_rc_t
1050 siena_filter_search(
1051         __in            siena_filter_tbl_t *sftp,
1052         __in            siena_filter_spec_t *spec,
1053         __in            uint32_t key,
1054         __in            boolean_t for_insert,
1055         __out           int *filter_index,
1056         __out           unsigned int *depth_required)
1057 {
1058         unsigned int hash, incr, filter_idx, depth;
1059
1060         hash = siena_filter_tbl_hash(key);
1061         incr = siena_filter_tbl_increment(key);
1062
1063         filter_idx = hash & (sftp->sft_size - 1);
1064         depth = 1;
1065
1066         for (;;) {
1067                 /*
1068                  * Return success if entry is used and matches this spec
1069                  * or entry is unused and we are trying to insert.
1070                  */
1071                 if (siena_filter_test_used(sftp, filter_idx) ?
1072                     siena_filter_equal(spec,
1073                     &sftp->sft_spec[filter_idx]) :
1074                     for_insert) {
1075                         *filter_index = filter_idx;
1076                         *depth_required = depth;
1077                         return (0);
1078                 }
1079
1080                 /* Return failure if we reached the maximum search depth */
1081                 if (depth == FILTER_CTL_SRCH_MAX)
1082                         return (for_insert ? EBUSY : ENOENT);
1083
1084                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1085                 ++depth;
1086         }
1087 }
1088
1089 static                  void
1090 siena_filter_clear_entry(
1091         __in            efx_nic_t *enp,
1092         __in            siena_filter_tbl_t *sftp,
1093         __in            int index)
1094 {
1095         efx_oword_t filter;
1096
1097         if (siena_filter_test_used(sftp, index)) {
1098                 siena_filter_clear_used(sftp, index);
1099
1100                 EFX_ZERO_OWORD(filter);
1101                 siena_filter_push_entry(enp,
1102                     sftp->sft_spec[index].sfs_type,
1103                     index, &filter);
1104
1105                 memset(&sftp->sft_spec[index],
1106                     0, sizeof (sftp->sft_spec[0]));
1107         }
1108 }
1109
1110                         void
1111 siena_filter_tbl_clear(
1112         __in            efx_nic_t *enp,
1113         __in            siena_filter_tbl_id_t tbl_id)
1114 {
1115         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1116         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1117         int index;
1118         efsys_lock_state_t state;
1119
1120         EFSYS_LOCK(enp->en_eslp, state);
1121
1122         for (index = 0; index < sftp->sft_size; ++index) {
1123                 siena_filter_clear_entry(enp, sftp, index);
1124         }
1125
1126         if (sftp->sft_used == 0)
1127                 siena_filter_reset_search_depth(sfp, tbl_id);
1128
1129         EFSYS_UNLOCK(enp->en_eslp, state);
1130 }
1131
1132 static  __checkReturn   efx_rc_t
1133 siena_filter_init(
1134         __in            efx_nic_t *enp)
1135 {
1136         siena_filter_t *sfp;
1137         siena_filter_tbl_t *sftp;
1138         int tbl_id;
1139         efx_rc_t rc;
1140
1141         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1142
1143         if (!sfp) {
1144                 rc = ENOMEM;
1145                 goto fail1;
1146         }
1147
1148         enp->en_filter.ef_siena_filter = sfp;
1149
1150         switch (enp->en_family) {
1151         case EFX_FAMILY_SIENA:
1152                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1153                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1154
1155                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1156                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1157
1158                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1159                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1160
1161                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1162                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1163                 break;
1164
1165         default:
1166                 rc = ENOTSUP;
1167                 goto fail2;
1168         }
1169
1170         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1171                 unsigned int bitmap_size;
1172
1173                 sftp = &sfp->sf_tbl[tbl_id];
1174                 if (sftp->sft_size == 0)
1175                         continue;
1176
1177                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1178                     sizeof (uint32_t));
1179                 bitmap_size =
1180                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1181
1182                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1183                 if (!sftp->sft_bitmap) {
1184                         rc = ENOMEM;
1185                         goto fail3;
1186                 }
1187
1188                 EFSYS_KMEM_ALLOC(enp->en_esip,
1189                     sftp->sft_size * sizeof (*sftp->sft_spec),
1190                     sftp->sft_spec);
1191                 if (!sftp->sft_spec) {
1192                         rc = ENOMEM;
1193                         goto fail4;
1194                 }
1195                 memset(sftp->sft_spec, 0,
1196                     sftp->sft_size * sizeof (*sftp->sft_spec));
1197         }
1198
1199         return (0);
1200
1201 fail4:
1202         EFSYS_PROBE(fail4);
1203
1204 fail3:
1205         EFSYS_PROBE(fail3);
1206
1207 fail2:
1208         EFSYS_PROBE(fail2);
1209         siena_filter_fini(enp);
1210
1211 fail1:
1212         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1213         return (rc);
1214 }
1215
1216 static                  void
1217 siena_filter_fini(
1218         __in            efx_nic_t *enp)
1219 {
1220         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1221         siena_filter_tbl_id_t tbl_id;
1222
1223         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1224         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1225
1226         if (sfp == NULL)
1227                 return;
1228
1229         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1230                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1231                 unsigned int bitmap_size;
1232
1233                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1234                     sizeof (uint32_t));
1235                 bitmap_size =
1236                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1237
1238                 if (sftp->sft_bitmap != NULL) {
1239                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1240                             sftp->sft_bitmap);
1241                         sftp->sft_bitmap = NULL;
1242                 }
1243
1244                 if (sftp->sft_spec != NULL) {
1245                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1246                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1247                         sftp->sft_spec = NULL;
1248                 }
1249         }
1250
1251         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1252             enp->en_filter.ef_siena_filter);
1253 }
1254
1255 /* Restore filter state after a reset */
1256 static  __checkReturn   efx_rc_t
1257 siena_filter_restore(
1258         __in            efx_nic_t *enp)
1259 {
1260         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1261         siena_filter_tbl_id_t tbl_id;
1262         siena_filter_tbl_t *sftp;
1263         siena_filter_spec_t *spec;
1264         efx_oword_t filter;
1265         int filter_idx;
1266         efsys_lock_state_t state;
1267         uint32_t key;
1268         efx_rc_t rc;
1269
1270         EFSYS_LOCK(enp->en_eslp, state);
1271
1272         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1273                 sftp = &sfp->sf_tbl[tbl_id];
1274                 for (filter_idx = 0;
1275                         filter_idx < sftp->sft_size;
1276                         filter_idx++) {
1277                         if (!siena_filter_test_used(sftp, filter_idx))
1278                                 continue;
1279
1280                         spec = &sftp->sft_spec[filter_idx];
1281                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1282                                 rc = EINVAL;
1283                                 goto fail1;
1284                         }
1285                         if ((rc = siena_filter_push_entry(enp,
1286                                     spec->sfs_type, filter_idx, &filter)) != 0)
1287                                 goto fail2;
1288                 }
1289         }
1290
1291         siena_filter_push_rx_limits(enp);
1292         siena_filter_push_tx_limits(enp);
1293
1294         EFSYS_UNLOCK(enp->en_eslp, state);
1295
1296         return (0);
1297
1298 fail2:
1299         EFSYS_PROBE(fail2);
1300
1301 fail1:
1302         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1303
1304         EFSYS_UNLOCK(enp->en_eslp, state);
1305
1306         return (rc);
1307 }
1308
1309 static   __checkReturn  efx_rc_t
1310 siena_filter_add(
1311         __in            efx_nic_t *enp,
1312         __inout         efx_filter_spec_t *spec,
1313         __in            boolean_t may_replace)
1314 {
1315         efx_rc_t rc;
1316         siena_filter_spec_t sf_spec;
1317         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1318         siena_filter_tbl_id_t tbl_id;
1319         siena_filter_tbl_t *sftp;
1320         siena_filter_spec_t *saved_sf_spec;
1321         efx_oword_t filter;
1322         int filter_idx;
1323         unsigned int depth;
1324         efsys_lock_state_t state;
1325         uint32_t key;
1326
1327
1328         EFSYS_ASSERT3P(spec, !=, NULL);
1329
1330         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1331                 goto fail1;
1332
1333         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1334         sftp = &sfp->sf_tbl[tbl_id];
1335
1336         if (sftp->sft_size == 0) {
1337                 rc = EINVAL;
1338                 goto fail2;
1339         }
1340
1341         key = siena_filter_build(&filter, &sf_spec);
1342
1343         EFSYS_LOCK(enp->en_eslp, state);
1344
1345         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1346             &filter_idx, &depth);
1347         if (rc != 0)
1348                 goto fail3;
1349
1350         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1351         saved_sf_spec = &sftp->sft_spec[filter_idx];
1352
1353         if (siena_filter_test_used(sftp, filter_idx)) {
1354                 if (may_replace == B_FALSE) {
1355                         rc = EEXIST;
1356                         goto fail4;
1357                 }
1358         }
1359         siena_filter_set_used(sftp, filter_idx);
1360         *saved_sf_spec = sf_spec;
1361
1362         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1363                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1364                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1365                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1366                         siena_filter_push_tx_limits(enp);
1367                 else
1368                         siena_filter_push_rx_limits(enp);
1369         }
1370
1371         siena_filter_push_entry(enp, sf_spec.sfs_type,
1372             filter_idx, &filter);
1373
1374         EFSYS_UNLOCK(enp->en_eslp, state);
1375         return (0);
1376
1377 fail4:
1378         EFSYS_PROBE(fail4);
1379
1380 fail3:
1381         EFSYS_UNLOCK(enp->en_eslp, state);
1382         EFSYS_PROBE(fail3);
1383
1384 fail2:
1385         EFSYS_PROBE(fail2);
1386
1387 fail1:
1388         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1389         return (rc);
1390 }
1391
1392 static   __checkReturn  efx_rc_t
1393 siena_filter_delete(
1394         __in            efx_nic_t *enp,
1395         __inout         efx_filter_spec_t *spec)
1396 {
1397         efx_rc_t rc;
1398         siena_filter_spec_t sf_spec;
1399         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1400         siena_filter_tbl_id_t tbl_id;
1401         siena_filter_tbl_t *sftp;
1402         efx_oword_t filter;
1403         int filter_idx;
1404         unsigned int depth;
1405         efsys_lock_state_t state;
1406         uint32_t key;
1407
1408         EFSYS_ASSERT3P(spec, !=, NULL);
1409
1410         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1411                 goto fail1;
1412
1413         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1414         sftp = &sfp->sf_tbl[tbl_id];
1415
1416         key = siena_filter_build(&filter, &sf_spec);
1417
1418         EFSYS_LOCK(enp->en_eslp, state);
1419
1420         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1421             &filter_idx, &depth);
1422         if (rc != 0)
1423                 goto fail2;
1424
1425         siena_filter_clear_entry(enp, sftp, filter_idx);
1426         if (sftp->sft_used == 0)
1427                 siena_filter_reset_search_depth(sfp, tbl_id);
1428
1429         EFSYS_UNLOCK(enp->en_eslp, state);
1430         return (0);
1431
1432 fail2:
1433         EFSYS_UNLOCK(enp->en_eslp, state);
1434         EFSYS_PROBE(fail2);
1435
1436 fail1:
1437         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1438         return (rc);
1439 }
1440
1441 #define SIENA_MAX_SUPPORTED_MATCHES 4
1442
1443 static  __checkReturn   efx_rc_t
1444 siena_filter_supported_filters(
1445         __in                            efx_nic_t *enp,
1446         __out_ecount(buffer_length)     uint32_t *buffer,
1447         __in                            size_t buffer_length,
1448         __out                           size_t *list_lengthp)
1449 {
1450         uint32_t index = 0;
1451         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1452         size_t list_length;
1453         efx_rc_t rc;
1454
1455         rx_matches[index++] =
1456             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1457             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1458             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1459
1460         rx_matches[index++] =
1461             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1462             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1463
1464         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1465                 rx_matches[index++] =
1466                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1467
1468                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1469         }
1470
1471         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1472         list_length = index;
1473
1474         *list_lengthp = list_length;
1475
1476         if (buffer_length < list_length) {
1477                 rc = ENOSPC;
1478                 goto fail1;
1479         }
1480
1481         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1482
1483         return (0);
1484
1485 fail1:
1486         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1487
1488         return (rc);
1489 }
1490
1491 #undef MAX_SUPPORTED
1492
1493 #endif /* EFSYS_OPT_SIENA */
1494
1495 #endif /* EFSYS_OPT_FILTER */