sys/dev/sfxge/common/efx_filter.c

   1 /*-
   2  * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25
  26 #include <sys/cdefs.h>
  27 __FBSDID("$FreeBSD$");
  28
  29 #include "efsys.h"
  30 #include "efx.h"
  31 #include "efx_types.h"
  32 #include "efx_regs.h"
  33 #include "efx_impl.h"
  34
  35
  36 #if EFSYS_OPT_FILTER
  37
  38 /* "Fudge factors" - difference between programmed value and actual depth.
  39  * Due to pipelined implementation we need to program H/W with a value that
  40  * is larger than the hop limit we want.
  41  */
  42 #define FILTER_CTL_SRCH_FUDGE_WILD 3
  43 #define FILTER_CTL_SRCH_FUDGE_FULL 1
  44
  45 /* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
  46  * We also need to avoid infinite loops in efx_filter_search() when the
  47  * table is full.
  48  */
  49 #define FILTER_CTL_SRCH_MAX 200
  50
  51 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  52  * key derived from the n-tuple. */
  53 static                  uint16_t
  54 efx_filter_tbl_hash(
  55         __in            uint32_t key)
  56 {
  57         uint16_t tmp;
  58
  59         /* First 16 rounds */
  60         tmp = 0x1fff ^ (uint16_t)(key >> 16);
  61         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  62         tmp = tmp ^ tmp >> 9;
  63
  64         /* Last 16 rounds */
  65         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
  66         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  67         tmp = tmp ^ tmp >> 9;
  68
  69         return (tmp);
  70 }
  71
  72
  73 /* To allow for hash collisions, filter search continues at these
  74  * increments from the first possible entry selected by the hash. */
  75 static                  uint16_t
  76 efx_filter_tbl_increment(
  77         __in            uint32_t key)
  78 {
  79         return ((uint16_t)(key * 2 - 1));
  80 }
  81
  82 static  __checkReturn   boolean_t
  83 efx_filter_test_used(
  84         __in            efx_filter_tbl_t *eftp,
  85         __in            unsigned int index)
  86 {
  87         EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
  88         return ((eftp->eft_bitmap[index / 32] & (1 << (index % 32))) != 0);
  89 }
  90
  91 static                  void
  92 efx_filter_set_used(
  93         __in            efx_filter_tbl_t *eftp,
  94         __in            unsigned int index)
  95 {
  96         EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
  97         eftp->eft_bitmap[index / 32] |= (1 << (index % 32));
  98         ++eftp->eft_used;
  99 }
 100
 101 static                  void
 102 efx_filter_clear_used(
 103         __in            efx_filter_tbl_t *eftp,
 104         __in            unsigned int index)
 105 {
 106         EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
 107         eftp->eft_bitmap[index / 32] &= ~(1 << (index % 32));
 108
 109         --eftp->eft_used;
 110         EFSYS_ASSERT3U(eftp->eft_used, >=, 0);
 111 }
 112
 113
 114 static                  efx_filter_tbl_id_t
 115 efx_filter_tbl_id(
 116         __in            efx_filter_type_t type)
 117 {
 118         efx_filter_tbl_id_t tbl_id;
 119
 120         switch (type)
 121         {
 122         case EFX_FILTER_RX_TCP_FULL:
 123         case EFX_FILTER_RX_TCP_WILD:
 124         case EFX_FILTER_RX_UDP_FULL:
 125         case EFX_FILTER_RX_UDP_WILD:
 126                 tbl_id = EFX_FILTER_TBL_RX_IP;
 127                 break;
 128
 129 #if EFSYS_OPT_SIENA
 130         case EFX_FILTER_RX_MAC_FULL:
 131         case EFX_FILTER_RX_MAC_WILD:
 132                 tbl_id = EFX_FILTER_TBL_RX_MAC;
 133                 break;
 134
 135         case EFX_FILTER_TX_TCP_FULL:
 136         case EFX_FILTER_TX_TCP_WILD:
 137         case EFX_FILTER_TX_UDP_FULL:
 138         case EFX_FILTER_TX_UDP_WILD:
 139                 tbl_id = EFX_FILTER_TBL_TX_IP;
 140                 break;
 141
 142         case EFX_FILTER_TX_MAC_FULL:
 143         case EFX_FILTER_TX_MAC_WILD:
 144                 tbl_id = EFX_FILTER_TBL_RX_MAC;
 145                 break;
 146 #endif  /* EFSYS_OPT_SIENA */
 147
 148         default:
 149                 EFSYS_ASSERT(B_FALSE);
 150                 break;
 151         }
 152         return (tbl_id);
 153 }
 154
 155 static                  void
 156 efx_filter_reset_search_depth(
 157         __inout         efx_filter_t *efp,
 158         __in            efx_filter_tbl_id_t tbl_id)
 159 {
 160         switch (tbl_id)
 161         {
 162         case EFX_FILTER_TBL_RX_IP:
 163                 efp->ef_depth[EFX_FILTER_RX_TCP_FULL] = 0;
 164                 efp->ef_depth[EFX_FILTER_RX_TCP_WILD] = 0;
 165                 efp->ef_depth[EFX_FILTER_RX_UDP_FULL] = 0;
 166                 efp->ef_depth[EFX_FILTER_RX_UDP_WILD] = 0;
 167                 break;
 168
 169 #if EFSYS_OPT_SIENA
 170         case EFX_FILTER_TBL_RX_MAC:
 171                 efp->ef_depth[EFX_FILTER_RX_MAC_FULL] = 0;
 172                 efp->ef_depth[EFX_FILTER_RX_MAC_WILD] = 0;
 173                 break;
 174
 175         case EFX_FILTER_TBL_TX_IP:
 176                 efp->ef_depth[EFX_FILTER_TX_TCP_FULL] = 0;
 177                 efp->ef_depth[EFX_FILTER_TX_TCP_WILD] = 0;
 178                 efp->ef_depth[EFX_FILTER_TX_UDP_FULL] = 0;
 179                 efp->ef_depth[EFX_FILTER_TX_UDP_WILD] = 0;
 180                 break;
 181
 182         case EFX_FILTER_TBL_TX_MAC:
 183                 efp->ef_depth[EFX_FILTER_TX_MAC_FULL] = 0;
 184                 efp->ef_depth[EFX_FILTER_TX_MAC_WILD] = 0;
 185                 break;
 186 #endif  /* EFSYS_OPT_SIENA */
 187
 188         default:
 189                 EFSYS_ASSERT(B_FALSE);
 190                 break;
 191         }
 192 }
 193
 194 static                  void
 195 efx_filter_push_rx_limits(
 196         __in            efx_nic_t *enp)
 197 {
 198         efx_filter_t *efp = &enp->en_filter;
 199         efx_oword_t oword;
 200
 201         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 202
 203         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 204             efp->ef_depth[EFX_FILTER_RX_TCP_FULL] +
 205             FILTER_CTL_SRCH_FUDGE_FULL);
 206         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 207             efp->ef_depth[EFX_FILTER_RX_TCP_WILD] +
 208             FILTER_CTL_SRCH_FUDGE_WILD);
 209         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 210             efp->ef_depth[EFX_FILTER_RX_UDP_FULL] +
 211             FILTER_CTL_SRCH_FUDGE_FULL);
 212         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 213             efp->ef_depth[EFX_FILTER_RX_UDP_WILD] +
 214             FILTER_CTL_SRCH_FUDGE_WILD);
 215
 216 #if EFSYS_OPT_SIENA
 217         if (efp->ef_tbl[EFX_FILTER_TBL_RX_MAC].eft_size) {
 218                 EFX_SET_OWORD_FIELD(oword,
 219                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 220                     efp->ef_depth[EFX_FILTER_RX_MAC_FULL] +
 221                     FILTER_CTL_SRCH_FUDGE_FULL);
 222                 EFX_SET_OWORD_FIELD(oword,
 223                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 224                     efp->ef_depth[EFX_FILTER_RX_MAC_WILD] +
 225                     FILTER_CTL_SRCH_FUDGE_WILD);
 226         }
 227 #endif /* EFSYS_OPT_SIENA */
 228
 229         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 230 }
 231
 232 static                  void
 233 efx_filter_push_tx_limits(
 234         __in            efx_nic_t *enp)
 235 {
 236         efx_filter_t *efp = &enp->en_filter;
 237         efx_oword_t oword;
 238
 239         if (efp->ef_tbl[EFX_FILTER_TBL_TX_IP].eft_size == 0)
 240                 return;
 241
 242         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 243
 244         EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 245             efp->ef_depth[EFX_FILTER_TX_TCP_FULL] +
 246             FILTER_CTL_SRCH_FUDGE_FULL);
 247         EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 248             efp->ef_depth[EFX_FILTER_TX_TCP_WILD] +
 249             FILTER_CTL_SRCH_FUDGE_WILD);
 250         EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 251             efp->ef_depth[EFX_FILTER_TX_UDP_FULL] +
 252             FILTER_CTL_SRCH_FUDGE_FULL);
 253         EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 254             efp->ef_depth[EFX_FILTER_TX_UDP_WILD] +
 255             FILTER_CTL_SRCH_FUDGE_WILD);
 256
 257         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 258 }
 259
 260 /* Build a filter entry and return its n-tuple key. */
 261 static  __checkReturn   uint32_t
 262 efx_filter_build(
 263         __out           efx_oword_t *filter,
 264         __in            efx_filter_spec_t *spec)
 265 {
 266         uint32_t dword3;
 267         uint32_t key;
 268         uint8_t  type  = spec->efs_type;
 269         uint8_t  flags = spec->efs_flags;
 270
 271         switch (efx_filter_tbl_id(type)) {
 272         case EFX_FILTER_TBL_RX_IP: {
 273                 boolean_t is_udp = (type == EFX_FILTER_RX_UDP_FULL ||
 274                     type == EFX_FILTER_RX_UDP_WILD);
 275                 EFX_POPULATE_OWORD_7(*filter,
 276                     FRF_BZ_RSS_EN,     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 277                     FRF_BZ_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 278                     FRF_AZ_TCP_UDP, is_udp,
 279                     FRF_AZ_RXQ_ID, spec->efs_dmaq_id,
 280                     EFX_DWORD_2, spec->efs_dword[2],
 281                     EFX_DWORD_1, spec->efs_dword[1],
 282                     EFX_DWORD_0, spec->efs_dword[0]);
 283                 dword3 = is_udp;
 284                 break;
 285         }
 286
 287 #if EFSYS_OPT_SIENA
 288         case EFX_FILTER_TBL_RX_MAC: {
 289                 boolean_t is_wild = (type == EFX_FILTER_RX_MAC_WILD);
 290                 EFX_POPULATE_OWORD_8(*filter,
 291                     FRF_CZ_RMFT_RSS_EN, (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 292                     FRF_CZ_RMFT_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 293                     FRF_CZ_RMFT_IP_OVERRIDE, (flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP) ? 1 : 0,
 294                     FRF_CZ_RMFT_RXQ_ID, spec->efs_dmaq_id,
 295                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 296                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->efs_dword[2],
 297                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->efs_dword[1],
 298                     FRF_CZ_RMFT_VLAN_ID, spec->efs_dword[0]);
 299                 dword3 = is_wild;
 300                 break;
 301         }
 302 #endif /* EFSYS_OPT_SIENA */
 303
 304         case EFX_FILTER_TBL_TX_IP: {
 305                 boolean_t is_udp = (type == EFX_FILTER_TX_UDP_FULL ||
 306                     type == EFX_FILTER_TX_UDP_WILD);
 307                 EFX_POPULATE_OWORD_5(*filter,
 308                     FRF_CZ_TIFT_TCP_UDP, is_udp,
 309                     FRF_CZ_TIFT_TXQ_ID, spec->efs_dmaq_id,
 310                     EFX_DWORD_2, spec->efs_dword[2],
 311                     EFX_DWORD_1, spec->efs_dword[1],
 312                     EFX_DWORD_0, spec->efs_dword[0]);
 313                 dword3 = is_udp | spec->efs_dmaq_id << 1;
 314                 break;
 315         }
 316
 317 #if EFSYS_OPT_SIENA
 318         case EFX_FILTER_TBL_TX_MAC: {
 319                 boolean_t is_wild = (type == EFX_FILTER_TX_MAC_WILD);
 320                 EFX_POPULATE_OWORD_5(*filter,
 321                     FRF_CZ_TMFT_TXQ_ID, spec->efs_dmaq_id,
 322                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 323                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->efs_dword[2],
 324                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->efs_dword[1],
 325                     FRF_CZ_TMFT_VLAN_ID, spec->efs_dword[0]);
 326                 dword3 = is_wild | spec->efs_dmaq_id << 1;
 327                 break;
 328         }
 329 #endif /* EFSYS_OPT_SIENA */
 330
 331         default:
 332                 EFSYS_ASSERT(B_FALSE);
 333         }
 334
 335         key = spec->efs_dword[0] ^ spec->efs_dword[1] ^ spec->efs_dword[2] ^ dword3;
 336         return (key);
 337 }
 338
 339 static  __checkReturn           int
 340 efx_filter_push_entry(
 341         __inout                 efx_nic_t *enp,
 342         __in                    efx_filter_type_t type,
 343         __in                    int index,
 344         __in                    efx_oword_t *eop)
 345 {
 346         int rc;
 347
 348         switch (type)
 349         {
 350         case EFX_FILTER_RX_TCP_FULL:
 351         case EFX_FILTER_RX_TCP_WILD:
 352         case EFX_FILTER_RX_UDP_FULL:
 353         case EFX_FILTER_RX_UDP_WILD:
 354                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index, eop);
 355                 break;
 356
 357 #if EFSYS_OPT_SIENA
 358         case EFX_FILTER_RX_MAC_FULL:
 359         case EFX_FILTER_RX_MAC_WILD:
 360                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index, eop);
 361                 break;
 362
 363         case EFX_FILTER_TX_TCP_FULL:
 364         case EFX_FILTER_TX_TCP_WILD:
 365         case EFX_FILTER_TX_UDP_FULL:
 366         case EFX_FILTER_TX_UDP_WILD:
 367                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index, eop);
 368                 break;
 369
 370         case EFX_FILTER_TX_MAC_FULL:
 371         case EFX_FILTER_TX_MAC_WILD:
 372                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index, eop);
 373                 break;
 374 #endif  /* EFSYS_OPT_SIENA */
 375
 376         default:
 377                 rc = ENOTSUP;
 378                 goto fail1;
 379         }
 380         return (0);
 381
 382 fail1:
 383         return (rc);
 384 }
 385
 386
 387 static  __checkReturn   boolean_t
 388 efx_filter_equal(
 389         __in            const efx_filter_spec_t *left,
 390         __in            const efx_filter_spec_t *right)
 391 {
 392         efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(left->efs_type);
 393
 394         if (left->efs_type != right->efs_type)
 395                 return (B_FALSE);
 396
 397         if (memcmp(left->efs_dword, right->efs_dword, sizeof(left->efs_dword)))
 398                 return (B_FALSE);
 399
 400         if ((tbl_id == EFX_FILTER_TBL_TX_IP ||
 401              tbl_id == EFX_FILTER_TBL_TX_MAC) &&
 402             left->efs_dmaq_id != right->efs_dmaq_id)
 403                 return (B_FALSE);
 404
 405         return (B_TRUE);
 406 }
 407
 408 static  __checkReturn   int
 409 efx_filter_search(
 410         __in            efx_filter_tbl_t *eftp,
 411         __in            efx_filter_spec_t *spec,
 412         __in            uint32_t key,
 413         __in            boolean_t for_insert,
 414         __out           int *filter_index,
 415         __out           int *depth_required)
 416 {
 417         unsigned hash, incr, filter_idx, depth;
 418
 419         hash = efx_filter_tbl_hash(key);
 420         incr = efx_filter_tbl_increment(key);
 421
 422         filter_idx = hash & (eftp->eft_size - 1);
 423         depth = 1;
 424
 425         for (;;) {
 426                 /* Return success if entry is used and matches this spec
 427                  * or entry is unused and we are trying to insert.
 428                  */
 429                 if (efx_filter_test_used(eftp, filter_idx) ?
 430                     efx_filter_equal(spec, &eftp->eft_spec[filter_idx]) :
 431                     for_insert) {
 432                         *filter_index = filter_idx;
 433                         *depth_required = depth;
 434                         return (0);
 435                 }
 436
 437                 /* Return failure if we reached the maximum search depth */
 438                 if (depth == FILTER_CTL_SRCH_MAX)
 439                         return for_insert ? EBUSY : ENOENT;
 440
 441                 filter_idx = (filter_idx + incr) & (eftp->eft_size - 1);
 442                 ++depth;
 443         }
 444 }
 445
 446         __checkReturn   int
 447 efx_filter_insert_filter(
 448         __in            efx_nic_t *enp,
 449         __in            efx_filter_spec_t *spec,
 450         __in            boolean_t replace)
 451 {
 452         efx_filter_t *efp = &enp->en_filter;
 453         efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);
 454         efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
 455         efx_filter_spec_t *saved_spec;
 456         efx_oword_t filter;
 457         int filter_idx;
 458         unsigned int depth;
 459         int state;
 460         uint32_t key;
 461         int rc;
 462
 463         if (eftp->eft_size == 0)
 464                 return (EINVAL);
 465
 466         key = efx_filter_build(&filter, spec);
 467
 468         EFSYS_LOCK(enp->en_eslp, state);
 469
 470         rc = efx_filter_search(eftp, spec, key, B_TRUE, &filter_idx, &depth);
 471         if (rc != 0)
 472                 goto done;
 473
 474         EFSYS_ASSERT3U(filter_idx, <, eftp->eft_size);
 475         saved_spec = &eftp->eft_spec[filter_idx];
 476
 477         if (efx_filter_test_used(eftp, filter_idx)) {
 478                 if (replace == B_FALSE) {
 479                         rc = EEXIST;
 480                         goto done;
 481                 }
 482         }
 483         efx_filter_set_used(eftp, filter_idx);
 484         *saved_spec = *spec;
 485
 486         if (efp->ef_depth[spec->efs_type] < depth) {
 487                 efp->ef_depth[spec->efs_type] = depth;
 488                 if (tbl_id == EFX_FILTER_TBL_TX_IP ||
 489                     tbl_id == EFX_FILTER_TBL_TX_MAC)
 490                         efx_filter_push_tx_limits(enp);
 491                 else
 492                         efx_filter_push_rx_limits(enp);
 493         }
 494
 495         efx_filter_push_entry(enp, spec->efs_type, filter_idx, &filter);
 496
 497 done:
 498         EFSYS_UNLOCK(enp->en_eslp, state);
 499         return (rc);
 500 }
 501
 502 static                  void
 503 efx_filter_clear_entry(
 504         __in            efx_nic_t *enp,
 505         __in            efx_filter_tbl_t *eftp,
 506         __in            int index)
 507 {
 508         efx_oword_t filter;
 509
 510         if (efx_filter_test_used(eftp, index)) {
 511                 efx_filter_clear_used(eftp, index);
 512
 513                 EFX_ZERO_OWORD(filter);
 514                 efx_filter_push_entry(enp, eftp->eft_spec[index].efs_type,
 515                     index, &filter);
 516
 517                 memset(&eftp->eft_spec[index], 0, sizeof(eftp->eft_spec[0]));
 518         }
 519 }
 520
 521         __checkReturn   int
 522 efx_filter_remove_filter(
 523         __in            efx_nic_t *enp,
 524         __in            efx_filter_spec_t *spec)
 525 {
 526         efx_filter_t *efp = &enp->en_filter;
 527         efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);
 528         efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
 529         efx_filter_spec_t *saved_spec;
 530         efx_oword_t filter;
 531         int filter_idx, depth;
 532         int state;
 533         uint32_t key;
 534         int rc;
 535
 536         key = efx_filter_build(&filter, spec);
 537
 538         EFSYS_LOCK(enp->en_eslp, state);
 539
 540         rc = efx_filter_search(eftp, spec, key, B_FALSE, &filter_idx, &depth);
 541         if (rc != 0)
 542                 goto out;
 543
 544         saved_spec = &eftp->eft_spec[filter_idx];
 545
 546         efx_filter_clear_entry(enp, eftp, filter_idx);
 547         if (eftp->eft_used == 0)
 548                 efx_filter_reset_search_depth(efp, tbl_id);
 549
 550         rc = 0;
 551
 552 out:
 553         EFSYS_UNLOCK(enp->en_eslp, state);
 554         return (rc);
 555 }
 556
 557                         void
 558 efx_filter_remove_index(
 559         __inout         efx_nic_t *enp,
 560         __in            efx_filter_type_t type,
 561         __in            int index)
 562 {
 563         efx_filter_t *efp = &enp->en_filter;
 564         enum efx_filter_tbl_id tbl_id = efx_filter_tbl_id(type);
 565         efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
 566         int state;
 567
 568         if (index < 0)
 569                 return;
 570
 571         EFSYS_LOCK(enp->en_eslp, state);
 572
 573         efx_filter_clear_entry(enp, eftp, index);
 574         if (eftp->eft_used == 0)
 575                 efx_filter_reset_search_depth(efp, tbl_id);
 576
 577         EFSYS_UNLOCK(enp->en_eslp, state);
 578 }
 579
 580                         void
 581 efx_filter_tbl_clear(
 582         __inout         efx_nic_t *enp,
 583         __in            efx_filter_tbl_id_t tbl_id)
 584 {
 585         efx_filter_t *efp = &enp->en_filter;
 586         efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
 587         int index;
 588         int state;
 589
 590         EFSYS_LOCK(enp->en_eslp, state);
 591
 592         for (index = 0; index < eftp->eft_size; ++index) {
 593                 efx_filter_clear_entry(enp, eftp, index);
 594         }
 595
 596         if (eftp->eft_used == 0)
 597                 efx_filter_reset_search_depth(efp, tbl_id);
 598
 599         EFSYS_UNLOCK(enp->en_eslp, state);
 600 }
 601
 602 /* Restore filter state after a reset */
 603                         void
 604 efx_filter_restore(
 605         __in            efx_nic_t *enp)
 606 {
 607         efx_filter_t *efp = &enp->en_filter;
 608         efx_filter_tbl_id_t tbl_id;
 609         efx_filter_tbl_t *eftp;
 610         efx_filter_spec_t *spec;
 611         efx_oword_t filter;
 612         int filter_idx;
 613         int state;
 614
 615         EFSYS_LOCK(enp->en_eslp, state);
 616
 617         for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
 618                 eftp = &efp->ef_tbl[tbl_id];
 619                 for (filter_idx = 0; filter_idx < eftp->eft_size; filter_idx++) {
 620                         if (!efx_filter_test_used(eftp, filter_idx))
 621                                 continue;
 622
 623                         spec = &eftp->eft_spec[filter_idx];
 624                         efx_filter_build(&filter, spec);
 625                         efx_filter_push_entry(enp, spec->efs_type,
 626                             filter_idx, &filter);
 627                 }
 628         }
 629
 630         efx_filter_push_rx_limits(enp);
 631         efx_filter_push_tx_limits(enp);
 632
 633         EFSYS_UNLOCK(enp->en_eslp, state);
 634 }
 635
 636                         void
 637 efx_filter_redirect_index(
 638         __inout         efx_nic_t *enp,
 639         __in            efx_filter_type_t type,
 640         __in            int filter_index,
 641         __in            int rxq_index)
 642 {
 643         efx_filter_t *efp = &enp->en_filter;
 644         efx_filter_tbl_t *eftp =
 645                 &efp->ef_tbl[efx_filter_tbl_id(type)];
 646         efx_filter_spec_t *spec;
 647         efx_oword_t filter;
 648         int state;
 649
 650         EFSYS_LOCK(enp->en_eslp, state);
 651
 652         spec = &eftp->eft_spec[filter_index];
 653         spec->efs_dmaq_id = (uint16_t)rxq_index;
 654
 655         efx_filter_build(&filter, spec);
 656         efx_filter_push_entry(enp, spec->efs_type, filter_index, &filter);
 657
 658         EFSYS_UNLOCK(enp->en_eslp, state);
 659 }
 660
 661         __checkReturn   int
 662 efx_filter_init(
 663         __in            efx_nic_t *enp)
 664 {
 665         efx_filter_t *efp = &enp->en_filter;
 666         efx_filter_tbl_t *eftp;
 667         int tbl_id;
 668         int rc;
 669
 670         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 671         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 672         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 673
 674         switch (enp->en_family)
 675         {
 676 #if EFSYS_OPT_FALCON
 677         case EFX_FAMILY_FALCON:
 678                 eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];
 679                 eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
 680                 break;
 681 #endif  /* EFSYS_OPT_FALCON */
 682
 683 #if EFSYS_OPT_SIENA
 684         case EFX_FAMILY_SIENA:
 685                 eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];
 686                 eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
 687
 688                 eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_MAC];
 689                 eftp->eft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
 690
 691                 eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_IP];
 692                 eftp->eft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
 693
 694                 eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_MAC];
 695                 eftp->eft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
 696                 break;
 697 #endif  /* EFSYS_OPT_SIENA */
 698
 699         default:
 700                 rc = ENOTSUP;
 701                 goto fail1;
 702         }
 703
 704         for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
 705                 unsigned int bitmap_size;
 706
 707                 eftp = &efp->ef_tbl[tbl_id];
 708                 if (eftp->eft_size == 0)
 709                         continue;
 710
 711                 EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));
 712                 bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;
 713
 714                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, eftp->eft_bitmap);
 715                 if (!eftp->eft_bitmap) {
 716                         rc = ENOMEM;
 717                         goto fail2;
 718                 }
 719
 720                 EFSYS_KMEM_ALLOC(enp->en_esip, eftp->eft_size * sizeof(*eftp->eft_spec),
 721                     eftp->eft_spec);
 722                 if (!eftp->eft_spec) {
 723                         rc = ENOMEM;
 724                         goto fail2;
 725                 }
 726                 memset(eftp->eft_spec, 0, eftp->eft_size * sizeof(*eftp->eft_spec));
 727         }
 728         enp->en_mod_flags |= EFX_MOD_FILTER;
 729
 730         return (0);
 731
 732 fail2:
 733         EFSYS_PROBE(fail2);
 734         efx_filter_fini(enp);
 735
 736 fail1:
 737         EFSYS_PROBE1(fail1, int, rc);
 738         return (rc);
 739 }
 740
 741                         void
 742 efx_filter_fini(
 743         __in            efx_nic_t *enp)
 744 {
 745         efx_filter_t *efp = &enp->en_filter;
 746         efx_filter_tbl_id_t tbl_id;
 747
 748         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 749         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 750
 751         for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
 752                 efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
 753                 unsigned int bitmap_size;
 754
 755                 EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));
 756                 bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;
 757
 758                 EFSYS_KMEM_FREE(enp->en_esip, bitmap_size, eftp->eft_bitmap);
 759                 eftp->eft_bitmap = NULL;
 760
 761                 EFSYS_KMEM_FREE(enp->en_esip, eftp->eft_size * sizeof(*eftp->eft_spec),
 762                     eftp->eft_spec);
 763                 eftp->eft_spec = NULL;
 764         }
 765
 766         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 767 }
 768
 769 extern                  void
 770 efx_filter_spec_rx_ipv4_tcp_full(
 771         __inout         efx_filter_spec_t *spec,
 772         __in            unsigned int flags,
 773         __in            uint32_t src_ip,
 774         __in            uint16_t src_tcp,
 775         __in            uint32_t dest_ip,
 776         __in            uint16_t dest_tcp)
 777 {
 778         EFSYS_ASSERT3P(spec, !=, NULL);
 779         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 780                     EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 781
 782         spec->efs_type = EFX_FILTER_RX_TCP_FULL;
 783         spec->efs_flags = (uint8_t)flags;
 784         spec->efs_dword[0] = src_tcp | src_ip << 16;
 785         spec->efs_dword[1] = dest_tcp << 16 | src_ip >> 16;
 786         spec->efs_dword[2] = dest_ip;
 787 }
 788
 789 extern                  void
 790 efx_filter_spec_rx_ipv4_tcp_wild(
 791         __inout         efx_filter_spec_t *spec,
 792         __in            unsigned int flags,
 793         __in            uint32_t dest_ip,
 794         __in            uint16_t dest_tcp)
 795 {
 796         EFSYS_ASSERT3P(spec, !=, NULL);
 797         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 798                     EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 799
 800         spec->efs_type = EFX_FILTER_RX_TCP_WILD;
 801         spec->efs_flags = (uint8_t)flags;
 802         spec->efs_dword[0] = 0;
 803         spec->efs_dword[1] = dest_tcp << 16;
 804         spec->efs_dword[2] = dest_ip;
 805 }
 806
 807 extern                  void
 808 efx_filter_spec_rx_ipv4_udp_full(
 809         __inout         efx_filter_spec_t *spec,
 810         __in            unsigned int flags,
 811         __in            uint32_t src_ip,
 812         __in            uint16_t src_udp,
 813         __in            uint32_t dest_ip,
 814         __in            uint16_t dest_udp)
 815 {
 816         EFSYS_ASSERT3P(spec, !=, NULL);
 817         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 818                     EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 819
 820         spec->efs_type = EFX_FILTER_RX_UDP_FULL;
 821         spec->efs_flags = (uint8_t)flags;
 822         spec->efs_dword[0] = src_udp | src_ip << 16;
 823         spec->efs_dword[1] = dest_udp << 16 | src_ip >> 16;
 824         spec->efs_dword[2] = dest_ip;
 825 }
 826
 827 extern                  void
 828 efx_filter_spec_rx_ipv4_udp_wild(
 829         __inout         efx_filter_spec_t *spec,
 830         __in            unsigned int flags,
 831         __in            uint32_t dest_ip,
 832         __in            uint16_t dest_udp)
 833 {
 834         EFSYS_ASSERT3P(spec, !=, NULL);
 835         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 836                     EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 837
 838         spec->efs_type = EFX_FILTER_RX_UDP_WILD;
 839         spec->efs_flags = (uint8_t)flags;
 840         spec->efs_dword[0] = dest_udp;
 841         spec->efs_dword[1] = 0;
 842         spec->efs_dword[2] = dest_ip;
 843 }
 844
 845 #if EFSYS_OPT_SIENA
 846 extern                  void
 847 efx_filter_spec_rx_mac_full(
 848         __inout         efx_filter_spec_t *spec,
 849         __in            unsigned int flags,
 850         __in            uint16_t vlan_id,
 851         __in            uint8_t *dest_mac)
 852 {
 853         EFSYS_ASSERT3P(spec, !=, NULL);
 854         EFSYS_ASSERT3P(dest_mac, !=, NULL);
 855         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 856                     EFX_FILTER_FLAG_RX_SCATTER |
 857                     EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);
 858
 859         spec->efs_type = EFX_FILTER_RX_MAC_FULL;
 860         spec->efs_flags = (uint8_t)flags;
 861         spec->efs_dword[0] = vlan_id;
 862         spec->efs_dword[1] =
 863             dest_mac[2] << 24 |
 864             dest_mac[3] << 16 |
 865             dest_mac[4] <<  8 |
 866             dest_mac[5];
 867         spec->efs_dword[2] =
 868             dest_mac[0] <<  8 |
 869             dest_mac[1];
 870 }
 871 #endif  /* EFSYS_OPT_SIENA */
 872
 873 #if EFSYS_OPT_SIENA
 874 extern                  void
 875 efx_filter_spec_rx_mac_wild(
 876         __inout         efx_filter_spec_t *spec,
 877         __in            unsigned int flags,
 878         __in            uint8_t *dest_mac)
 879 {
 880         EFSYS_ASSERT3P(spec, !=, NULL);
 881         EFSYS_ASSERT3P(dest_mac, !=, NULL);
 882         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 883                     EFX_FILTER_FLAG_RX_SCATTER |
 884                     EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);
 885
 886         spec->efs_type = EFX_FILTER_RX_MAC_WILD;
 887         spec->efs_flags = (uint8_t)flags;
 888         spec->efs_dword[0] = 0;
 889         spec->efs_dword[1] =
 890             dest_mac[2] << 24 |
 891             dest_mac[3] << 16 |
 892             dest_mac[4] <<  8 |
 893             dest_mac[5];
 894         spec->efs_dword[2] =
 895             dest_mac[0] <<  8 |
 896             dest_mac[1];
 897 }
 898 #endif  /* EFSYS_OPT_SIENA */
 899
 900 #if EFSYS_OPT_SIENA
 901 extern                  void
 902 efx_filter_spec_tx_ipv4_tcp_full(
 903         __inout         efx_filter_spec_t *spec,
 904         __in            uint32_t src_ip,
 905         __in            uint16_t src_tcp,
 906         __in            uint32_t dest_ip,
 907         __in            uint16_t dest_tcp)
 908 {
 909         EFSYS_ASSERT3P(spec, !=, NULL);
 910
 911         spec->efs_type = EFX_FILTER_TX_TCP_FULL;
 912         spec->efs_flags = 0;
 913         spec->efs_dword[0] = src_tcp | src_ip << 16;
 914         spec->efs_dword[1] = dest_tcp << 16 | src_ip >> 16;
 915         spec->efs_dword[2] = dest_ip;
 916 }
 917 #endif  /* EFSYS_OPT_SIENA */
 918
 919 #if EFSYS_OPT_SIENA
 920 extern                  void
 921 efx_filter_spec_tx_ipv4_tcp_wild(
 922         __inout         efx_filter_spec_t *spec,
 923         __in            uint32_t src_ip,
 924         __in            uint16_t src_tcp)
 925 {
 926         EFSYS_ASSERT3P(spec, !=, NULL);
 927
 928         spec->efs_type = EFX_FILTER_TX_TCP_WILD;
 929         spec->efs_flags = 0;
 930         spec->efs_dword[0] = 0;
 931         spec->efs_dword[1] = src_tcp << 16;
 932         spec->efs_dword[2] = src_ip;
 933 }
 934 #endif  /* EFSYS_OPT_SIENA */
 935
 936 #if EFSYS_OPT_SIENA
 937 extern                  void
 938 efx_filter_spec_tx_ipv4_udp_full(
 939         __inout         efx_filter_spec_t *spec,
 940         __in            uint32_t src_ip,
 941         __in            uint16_t src_udp,
 942         __in            uint32_t dest_ip,
 943         __in            uint16_t dest_udp)
 944 {
 945         EFSYS_ASSERT3P(spec, !=, NULL);
 946
 947         spec->efs_type = EFX_FILTER_TX_UDP_FULL;
 948         spec->efs_flags = 0;
 949         spec->efs_dword[0] = src_udp | src_ip << 16;
 950         spec->efs_dword[1] = dest_udp << 16 | src_ip >> 16;
 951         spec->efs_dword[2] = dest_ip;
 952 }
 953 #endif  /* EFSYS_OPT_SIENA */
 954
 955 #if EFSYS_OPT_SIENA
 956 extern                  void
 957 efx_filter_spec_tx_ipv4_udp_wild(
 958         __inout         efx_filter_spec_t *spec,
 959         __in            uint32_t src_ip,
 960         __in            uint16_t src_udp)
 961 {
 962         EFSYS_ASSERT3P(spec, !=, NULL);
 963
 964         spec->efs_type = EFX_FILTER_TX_UDP_WILD;
 965         spec->efs_flags = 0;
 966         spec->efs_dword[0] = src_udp;
 967         spec->efs_dword[1] = 0;
 968         spec->efs_dword[2] = src_ip;
 969 }
 970 #endif  /* EFSYS_OPT_SIENA */
 971
 972 #if EFSYS_OPT_SIENA
 973 extern                  void
 974 efx_filter_spec_tx_mac_full(
 975         __inout         efx_filter_spec_t *spec,
 976         __in            uint16_t vlan_id,
 977         __in            uint8_t *src_mac)
 978 {
 979         EFSYS_ASSERT3P(spec, !=, NULL);
 980         EFSYS_ASSERT3P(src_mac, !=, NULL);
 981
 982         spec->efs_type = EFX_FILTER_TX_MAC_FULL;
 983         spec->efs_flags = 0;
 984         spec->efs_dword[0] = vlan_id;
 985         spec->efs_dword[1] =
 986             src_mac[2] << 24 |
 987             src_mac[3] << 16 |
 988             src_mac[4] <<  8 |
 989             src_mac[5];
 990         spec->efs_dword[2] =
 991             src_mac[0] <<  8 |
 992             src_mac[1];
 993 }
 994 #endif  /* EFSYS_OPT_SIENA */
 995
 996 #if EFSYS_OPT_SIENA
 997 extern                  void
 998 efx_filter_spec_tx_mac_wild(
 999         __inout         efx_filter_spec_t *spec,
1000         __in            uint8_t *src_mac)
1001 {
1002         EFSYS_ASSERT3P(spec, !=, NULL);
1003         EFSYS_ASSERT3P(src_mac, !=, NULL);
1004
1005         spec->efs_type = EFX_FILTER_TX_MAC_WILD;
1006         spec->efs_flags = 0;
1007         spec->efs_dword[0] = 0;
1008         spec->efs_dword[1] =
1009             src_mac[2] << 24 |
1010             src_mac[3] << 16 |
1011             src_mac[4] <<  8 |
1012             src_mac[5];
1013         spec->efs_dword[2] =
1014             src_mac[0] <<  8 |
1015             src_mac[1];
1016 }
1017 #endif  /* EFSYS_OPT_SIENA */
1018
1019
1020 #endif /* EFSYS_OPT_FILTER */