sys/dev/sfxge/common/efx_rx.c

   1 /*-
   2  * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25
  26 #include <sys/cdefs.h>
  27 __FBSDID("$FreeBSD$");
  28
  29 #include "efsys.h"
  30 #include "efx.h"
  31 #include "efx_types.h"
  32 #include "efx_regs.h"
  33 #include "efx_impl.h"
  34
  35         __checkReturn   int
  36 efx_rx_init(
  37         __in            efx_nic_t *enp)
  38 {
  39         efx_oword_t oword;
  40         unsigned int index;
  41         int rc;
  42
  43         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
  44         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
  45
  46         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
  47                 rc = EINVAL;
  48                 goto fail1;
  49         }
  50
  51         if (enp->en_mod_flags & EFX_MOD_RX) {
  52                 rc = EINVAL;
  53                 goto fail2;
  54         }
  55
  56         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
  57
  58         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
  59         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
  60         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
  61         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
  62         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
  63         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
  64         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
  65
  66         /* Zero the RSS table */
  67         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
  68             index++) {
  69                 EFX_ZERO_OWORD(oword);
  70                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
  71                                     index, &oword);
  72         }
  73
  74         enp->en_mod_flags |= EFX_MOD_RX;
  75         return (0);
  76
  77 fail2:
  78         EFSYS_PROBE(fail2);
  79 fail1:
  80         EFSYS_PROBE1(fail1, int, rc);
  81
  82         return (rc);
  83 }
  84
  85 #if EFSYS_OPT_RX_HDR_SPLIT
  86         __checkReturn   int
  87 efx_rx_hdr_split_enable(
  88         __in            efx_nic_t *enp,
  89         __in            unsigned int hdr_buf_size,
  90         __in            unsigned int pld_buf_size)
  91 {
  92         unsigned int nhdr32;
  93         unsigned int npld32;
  94         efx_oword_t oword;
  95         int rc;
  96
  97         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
  98         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
  99         EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
 100
 101         nhdr32 = hdr_buf_size / 32;
 102         if ((nhdr32 == 0) ||
 103             (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
 104             ((hdr_buf_size % 32) != 0)) {
 105                 rc = EINVAL;
 106                 goto fail1;
 107         }
 108
 109         npld32 = pld_buf_size / 32;
 110         if ((npld32 == 0) ||
 111             (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
 112             ((pld_buf_size % 32) != 0)) {
 113                 rc = EINVAL;
 114                 goto fail2;
 115         }
 116
 117         if (enp->en_rx_qcount > 0) {
 118                 rc = EBUSY;
 119                 goto fail3;
 120         }
 121
 122         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 123
 124         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
 125         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
 126         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
 127
 128         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 129
 130         return (0);
 131
 132 fail3:
 133         EFSYS_PROBE(fail3);
 134 fail2:
 135         EFSYS_PROBE(fail2);
 136 fail1:
 137         EFSYS_PROBE1(fail1, int, rc);
 138
 139         return (rc);
 140 }
 141 #endif  /* EFSYS_OPT_RX_HDR_SPLIT */
 142
 143
 144 #if EFSYS_OPT_RX_SCATTER
 145         __checkReturn   int
 146 efx_rx_scatter_enable(
 147         __in            efx_nic_t *enp,
 148         __in            unsigned int buf_size)
 149 {
 150         unsigned int nbuf32;
 151         efx_oword_t oword;
 152         int rc;
 153
 154         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 155         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 156         EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
 157
 158         nbuf32 = buf_size / 32;
 159         if ((nbuf32 == 0) ||
 160             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 161             ((buf_size % 32) != 0)) {
 162                 rc = EINVAL;
 163                 goto fail1;
 164         }
 165
 166         if (enp->en_rx_qcount > 0) {
 167                 rc = EBUSY;
 168                 goto fail2;
 169         }
 170
 171         /* Set scatter buffer size */
 172         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 173         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 174         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 175
 176         /* Enable scatter for packets not matching a filter */
 177         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 178         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 179         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 180
 181         return (0);
 182
 183 fail2:
 184         EFSYS_PROBE(fail2);
 185 fail1:
 186         EFSYS_PROBE1(fail1, int, rc);
 187
 188         return (rc);
 189 }
 190 #endif  /* EFSYS_OPT_RX_SCATTER */
 191
 192
 193 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 194         do {                                                            \
 195                 efx_oword_t oword;                                      \
 196                                                                         \
 197                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 198                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 199                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 200                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 201                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 202                     (_insert) ? 1 : 0);                                 \
 203                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 204                                                                         \
 205                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 206                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 207                             &oword);                                    \
 208                         EFX_SET_OWORD_FIELD(oword,                      \
 209                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 210                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 211                             &oword);                                    \
 212                 }                                                       \
 213                                                                         \
 214                 _NOTE(CONSTANTCONDITION)                                \
 215         } while (B_FALSE)
 216
 217 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 218         do {                                                            \
 219                 efx_oword_t oword;                                      \
 220                                                                         \
 221                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 222                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 223                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 224                     (_ip) ? 1 : 0);                                     \
 225                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 226                     (_tcp) ? 0 : 1);                                    \
 227                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 228                     (_insert) ? 1 : 0);                                 \
 229                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 230                                                                         \
 231                 _NOTE(CONSTANTCONDITION)                                \
 232         } while (B_FALSE)
 233
 234 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 235         do {                                                            \
 236                 efx_oword_t oword;                                      \
 237                                                                         \
 238                 if ((_enp)->en_family == EFX_FAMILY_FALCON) {           \
 239                         (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0;        \
 240                         break;                                          \
 241                 }                                                       \
 242                                                                         \
 243                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 244                 EFX_SET_OWORD_FIELD(oword,                              \
 245                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 246                 EFX_SET_OWORD_FIELD(oword,                              \
 247                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 248                 EFX_SET_OWORD_FIELD(oword,                              \
 249                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 250                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 251                                                                         \
 252                 (_rc) = 0;                                              \
 253                                                                         \
 254                 _NOTE(CONSTANTCONDITION)                                \
 255         } while (B_FALSE)
 256
 257
 258 #if EFSYS_OPT_RX_SCALE
 259         __checkReturn   int
 260 efx_rx_scale_mode_set(
 261         __in            efx_nic_t *enp,
 262         __in            efx_rx_hash_alg_t alg,
 263         __in            efx_rx_hash_type_t type,
 264         __in            boolean_t insert)
 265 {
 266         int rc;
 267
 268         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 269         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 270         EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_FALCON);
 271
 272         switch (alg) {
 273         case EFX_RX_HASHALG_LFSR:
 274                 EFX_RX_LFSR_HASH(enp, insert);
 275                 break;
 276
 277         case EFX_RX_HASHALG_TOEPLITZ:
 278                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 279                     type & (1 << EFX_RX_HASH_IPV4),
 280                     type & (1 << EFX_RX_HASH_TCPIPV4));
 281
 282                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 283                     type & (1 << EFX_RX_HASH_IPV6),
 284                     type & (1 << EFX_RX_HASH_TCPIPV6),
 285                     rc);
 286                 if (rc != 0)
 287                         goto fail1;
 288
 289                 break;
 290
 291         default:
 292                 rc = EINVAL;
 293                 goto fail2;
 294         }
 295
 296         return (0);
 297
 298 fail2:
 299         EFSYS_PROBE(fail2);
 300 fail1:
 301         EFSYS_PROBE1(fail1, int, rc);
 302
 303         EFX_RX_LFSR_HASH(enp, B_FALSE);
 304
 305         return (rc);
 306 }
 307 #endif
 308
 309 #if EFSYS_OPT_RX_SCALE
 310         __checkReturn   int
 311 efx_rx_scale_toeplitz_ipv4_key_set(
 312         __in            efx_nic_t *enp,
 313         __in_ecount(n)  uint8_t *key,
 314         __in            size_t n)
 315 {
 316         efx_oword_t oword;
 317         unsigned int byte;
 318         unsigned int offset;
 319         int rc;
 320
 321         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 322         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 323
 324         byte = 0;
 325
 326         /* Write toeplitz hash key */
 327         EFX_ZERO_OWORD(oword);
 328         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 329             offset > 0 && byte < n;
 330             --offset)
 331                 oword.eo_u8[offset - 1] = key[byte++];
 332
 333         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 334
 335         byte = 0;
 336
 337         /* Verify toeplitz hash key */
 338         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 339         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 340             offset > 0 && byte < n;
 341             --offset) {
 342                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 343                         rc = EFAULT;
 344                         goto fail1;
 345                 }
 346         }
 347
 348         return (0);
 349
 350 fail1:
 351         EFSYS_PROBE1(fail1, int, rc);
 352
 353         return (rc);
 354 }
 355 #endif
 356
 357 #if EFSYS_OPT_RX_SCALE
 358         __checkReturn   int
 359 efx_rx_scale_toeplitz_ipv6_key_set(
 360         __in            efx_nic_t *enp,
 361         __in_ecount(n)  uint8_t *key,
 362         __in            size_t n)
 363 {
 364         efx_oword_t oword;
 365         unsigned int byte;
 366         int offset;
 367         int rc;
 368
 369         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 370         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 371
 372         byte = 0;
 373
 374         /* Write toeplitz hash key 3 */
 375         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 376         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 377             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 378             offset > 0 && byte < n;
 379             --offset)
 380                 oword.eo_u8[offset - 1] = key[byte++];
 381
 382         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 383
 384         /* Write toeplitz hash key 2 */
 385         EFX_ZERO_OWORD(oword);
 386         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 387             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 388             offset > 0 && byte < n;
 389             --offset)
 390                 oword.eo_u8[offset - 1] = key[byte++];
 391
 392         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 393
 394         /* Write toeplitz hash key 1 */
 395         EFX_ZERO_OWORD(oword);
 396         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 397             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 398             offset > 0 && byte < n;
 399             --offset)
 400                 oword.eo_u8[offset - 1] = key[byte++];
 401
 402         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 403
 404         byte = 0;
 405
 406         /* Verify toeplitz hash key 3 */
 407         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 408         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 409             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 410             offset > 0 && byte < n;
 411             --offset) {
 412                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 413                         rc = EFAULT;
 414                         goto fail1;
 415                 }
 416         }
 417
 418         /* Verify toeplitz hash key 2 */
 419         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 420         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 421             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 422             offset > 0 && byte < n;
 423             --offset) {
 424                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 425                         rc = EFAULT;
 426                         goto fail2;
 427                 }
 428         }
 429
 430         /* Verify toeplitz hash key 1 */
 431         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 432         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 433             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 434             offset > 0 && byte < n;
 435             --offset) {
 436                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 437                         rc = EFAULT;
 438                         goto fail3;
 439                 }
 440         }
 441
 442         return (0);
 443
 444 fail3:
 445         EFSYS_PROBE(fail3);
 446 fail2:
 447         EFSYS_PROBE(fail2);
 448 fail1:
 449         EFSYS_PROBE1(fail1, int, rc);
 450
 451         return (rc);
 452 }
 453 #endif
 454
 455 #if EFSYS_OPT_RX_SCALE
 456         __checkReturn   int
 457 efx_rx_scale_tbl_set(
 458         __in            efx_nic_t *enp,
 459         __in_ecount(n)  unsigned int *table,
 460         __in            size_t n)
 461 {
 462         efx_oword_t oword;
 463         int index;
 464         int rc;
 465
 466         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 467         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 468
 469         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
 470         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
 471
 472         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
 473                 rc = EINVAL;
 474                 goto fail1;
 475         }
 476
 477         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
 478                 uint32_t byte;
 479
 480                 /* Calculate the entry to place in the table */
 481                 byte = (uint32_t)table[index % n];
 482
 483                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
 484
 485                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
 486
 487                 /* Write the table */
 488                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 489                                     index, &oword);
 490         }
 491
 492         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
 493                 uint32_t byte;
 494
 495                 /* Determine if we're starting a new batch */
 496                 byte = (uint32_t)table[index % n];
 497
 498                 /* Read the table */
 499                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
 500                                     index, &oword);
 501
 502                 /* Verify the entry */
 503                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
 504                         rc = EFAULT;
 505                         goto fail2;
 506                 }
 507         }
 508
 509         return (0);
 510
 511 fail2:
 512         EFSYS_PROBE(fail2);
 513 fail1:
 514         EFSYS_PROBE1(fail1, int, rc);
 515
 516         return (rc);
 517 }
 518 #endif
 519
 520 #if EFSYS_OPT_FILTER
 521 extern  __checkReturn   int
 522 efx_rx_filter_insert(
 523         __in            efx_rxq_t *erp,
 524         __inout         efx_filter_spec_t *spec)
 525 {
 526         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 527         EFSYS_ASSERT3P(spec, !=, NULL);
 528
 529         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 530         return efx_filter_insert_filter(erp->er_enp, spec, B_FALSE);
 531 }
 532 #endif
 533
 534 #if EFSYS_OPT_FILTER
 535 extern  __checkReturn   int
 536 efx_rx_filter_remove(
 537         __in            efx_rxq_t *erp,
 538         __inout         efx_filter_spec_t *spec)
 539 {
 540         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 541         EFSYS_ASSERT3P(spec, !=, NULL);
 542
 543         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 544         return efx_filter_remove_filter(erp->er_enp, spec);
 545 }
 546 #endif
 547
 548 extern                  void
 549 efx_rx_qpost(
 550         __in            efx_rxq_t *erp,
 551         __in_ecount(n)  efsys_dma_addr_t *addrp,
 552         __in            size_t size,
 553         __in            unsigned int n,
 554         __in            unsigned int completed,
 555         __in            unsigned int added)
 556 {
 557         efx_qword_t qword;
 558         unsigned int i;
 559         unsigned int offset;
 560         unsigned int id;
 561
 562         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 563
 564         /* The client driver must not overfill the queue */
 565         EFSYS_ASSERT3U(added - completed + n, <=,
 566             EFX_RXQ_LIMIT(erp->er_mask + 1));
 567
 568         id = added & (erp->er_mask);
 569         for (i = 0; i < n; i++) {
 570                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 571                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 572                     size_t, size);
 573
 574                 EFX_POPULATE_QWORD_3(qword,
 575                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
 576                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
 577                     (uint32_t)(addrp[i] & 0xffffffff),
 578                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
 579                     (uint32_t)(addrp[i] >> 32));
 580
 581                 offset = id * sizeof (efx_qword_t);
 582                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 583
 584                 id = (id + 1) & (erp->er_mask);
 585         }
 586 }
 587
 588                 void
 589 efx_rx_qpush(
 590         __in    efx_rxq_t *erp,
 591         __in    unsigned int added)
 592 {
 593         efx_nic_t *enp = erp->er_enp;
 594         uint32_t wptr;
 595         efx_oword_t oword;
 596         efx_dword_t dword;
 597
 598         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 599
 600         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 601         EFSYS_PIO_WRITE_BARRIER();
 602
 603         /* Push the populated descriptors out */
 604         wptr = added & erp->er_mask;
 605
 606         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
 607
 608         /* Only write the third DWORD */
 609         EFX_POPULATE_DWORD_1(dword,
 610             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
 611         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
 612                             erp->er_index, &dword, B_FALSE);
 613 }
 614
 615                 void
 616 efx_rx_qflush(
 617         __in    efx_rxq_t *erp)
 618 {
 619         efx_nic_t *enp = erp->er_enp;
 620         efx_oword_t oword;
 621         uint32_t label;
 622
 623         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 624
 625         label = erp->er_index;
 626
 627         /* Flush the queue */
 628         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 629             FRF_AZ_RX_FLUSH_DESCQ, label);
 630         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
 631 }
 632
 633                 void
 634 efx_rx_qenable(
 635         __in    efx_rxq_t *erp)
 636 {
 637         efx_nic_t *enp = erp->er_enp;
 638         efx_oword_t oword;
 639
 640         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 641
 642         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
 643                             erp->er_index, &oword);
 644
 645         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
 646         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
 647         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
 648
 649         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 650                             erp->er_index, &oword);
 651 }
 652
 653         __checkReturn   int
 654 efx_rx_qcreate(
 655         __in            efx_nic_t *enp,
 656         __in            unsigned int index,
 657         __in            unsigned int label,
 658         __in            efx_rxq_type_t type,
 659         __in            efsys_mem_t *esmp,
 660         __in            size_t n,
 661         __in            uint32_t id,
 662         __in            efx_evq_t *eep,
 663         __deref_out     efx_rxq_t **erpp)
 664 {
 665         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 666         efx_rxq_t *erp;
 667         efx_oword_t oword;
 668         uint32_t size;
 669         boolean_t split;
 670         boolean_t jumbo;
 671         int rc;
 672
 673         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 674         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 675
 676         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
 677         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 678         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 679
 680         if (!ISP2(n) || !(n & EFX_RXQ_NDESCS_MASK)) {
 681                 rc = EINVAL;
 682                 goto fail1;
 683         }
 684         if (index >= encp->enc_rxq_limit) {
 685                 rc = EINVAL;
 686                 goto fail2;
 687         }
 688         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
 689             size++)
 690                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
 691                         break;
 692         if (id + (1 << size) >= encp->enc_buftbl_limit) {
 693                 rc = EINVAL;
 694                 goto fail3;
 695         }
 696
 697         switch (type) {
 698         case EFX_RXQ_TYPE_DEFAULT:
 699                 split = B_FALSE;
 700                 jumbo = B_FALSE;
 701                 break;
 702
 703 #if EFSYS_OPT_RX_HDR_SPLIT
 704         case EFX_RXQ_TYPE_SPLIT_HEADER:
 705                 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
 706                         rc = EINVAL;
 707                         goto fail4;
 708                 }
 709                 split = B_TRUE;
 710                 jumbo = B_TRUE;
 711                 break;
 712
 713         case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
 714                 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
 715                         rc = EINVAL;
 716                         goto fail4;
 717                 }
 718                 split = B_FALSE;
 719                 jumbo = B_TRUE;
 720                 break;
 721 #endif  /* EFSYS_OPT_RX_HDR_SPLIT */
 722
 723 #if EFSYS_OPT_RX_SCATTER
 724         case EFX_RXQ_TYPE_SCATTER:
 725                 if (enp->en_family < EFX_FAMILY_SIENA) {
 726                         rc = EINVAL;
 727                         goto fail4;
 728                 }
 729                 split = B_FALSE;
 730                 jumbo = B_TRUE;
 731                 break;
 732 #endif  /* EFSYS_OPT_RX_SCATTER */
 733
 734         default:
 735                 rc = EINVAL;
 736                 goto fail4;
 737         }
 738
 739         /* Allocate an RXQ object */
 740         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 741
 742         if (erp == NULL) {
 743                 rc = ENOMEM;
 744                 goto fail5;
 745         }
 746
 747         erp->er_magic = EFX_RXQ_MAGIC;
 748         erp->er_enp = enp;
 749         erp->er_index = index;
 750         erp->er_mask = n - 1;
 751         erp->er_esmp = esmp;
 752
 753         /* Set up the new descriptor queue */
 754         EFX_POPULATE_OWORD_10(oword,
 755             FRF_CZ_RX_HDR_SPLIT, split,
 756             FRF_AZ_RX_ISCSI_DDIG_EN, 0,
 757             FRF_AZ_RX_ISCSI_HDIG_EN, 0,
 758             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
 759             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
 760             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 761             FRF_AZ_RX_DESCQ_LABEL, label,
 762             FRF_AZ_RX_DESCQ_SIZE, size,
 763             FRF_AZ_RX_DESCQ_TYPE, 0,
 764             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
 765
 766         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 767                             erp->er_index, &oword);
 768
 769         enp->en_rx_qcount++;
 770         *erpp = erp;
 771         return (0);
 772
 773 fail5:
 774         EFSYS_PROBE(fail5);
 775 fail4:
 776         EFSYS_PROBE(fail4);
 777 fail3:
 778         EFSYS_PROBE(fail3);
 779 fail2:
 780         EFSYS_PROBE(fail2);
 781 fail1:
 782         EFSYS_PROBE1(fail1, int, rc);
 783
 784         return (rc);
 785 }
 786
 787                 void
 788 efx_rx_qdestroy(
 789         __in    efx_rxq_t *erp)
 790 {
 791         efx_nic_t *enp = erp->er_enp;
 792         efx_oword_t oword;
 793
 794         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 795
 796         EFSYS_ASSERT(enp->en_rx_qcount != 0);
 797         --enp->en_rx_qcount;
 798
 799         /* Purge descriptor queue */
 800         EFX_ZERO_OWORD(oword);
 801
 802         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 803                             erp->er_index, &oword);
 804
 805         /* Free the RXQ object */
 806         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 807 }
 808
 809                 void
 810 efx_rx_fini(
 811         __in    efx_nic_t *enp)
 812 {
 813         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 814         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 815         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 816         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 817
 818         enp->en_mod_flags &= ~EFX_MOD_RX;
 819 }