sys/dev/sfxge/common/efx_rx.c

   1 /*-
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "efx.h"
  35 #include "efx_impl.h"
  36
  37
  38 #if EFSYS_OPT_SIENA
  39
  40 static  __checkReturn   efx_rc_t
  41 siena_rx_init(
  42         __in            efx_nic_t *enp);
  43
  44 static                  void
  45 siena_rx_fini(
  46         __in            efx_nic_t *enp);
  47
  48 #if EFSYS_OPT_RX_SCATTER
  49 static  __checkReturn   efx_rc_t
  50 siena_rx_scatter_enable(
  51         __in            efx_nic_t *enp,
  52         __in            unsigned int buf_size);
  53 #endif /* EFSYS_OPT_RX_SCATTER */
  54
  55 #if EFSYS_OPT_RX_SCALE
  56 static  __checkReturn   efx_rc_t
  57 siena_rx_scale_mode_set(
  58         __in            efx_nic_t *enp,
  59         __in            efx_rx_hash_alg_t alg,
  60         __in            efx_rx_hash_type_t type,
  61         __in            boolean_t insert);
  62
  63 static  __checkReturn   efx_rc_t
  64 siena_rx_scale_key_set(
  65         __in            efx_nic_t *enp,
  66         __in_ecount(n)  uint8_t *key,
  67         __in            size_t n);
  68
  69 static  __checkReturn   efx_rc_t
  70 siena_rx_scale_tbl_set(
  71         __in            efx_nic_t *enp,
  72         __in_ecount(n)  unsigned int *table,
  73         __in            size_t n);
  74
  75 static  __checkReturn   uint32_t
  76 siena_rx_prefix_hash(
  77         __in            efx_nic_t *enp,
  78         __in            efx_rx_hash_alg_t func,
  79         __in            uint8_t *buffer);
  80
  81 #endif /* EFSYS_OPT_RX_SCALE */
  82
  83 static  __checkReturn   efx_rc_t
  84 siena_rx_prefix_pktlen(
  85         __in            efx_nic_t *enp,
  86         __in            uint8_t *buffer,
  87         __out           uint16_t *lengthp);
  88
  89 static                  void
  90 siena_rx_qpost(
  91         __in            efx_rxq_t *erp,
  92         __in_ecount(n)  efsys_dma_addr_t *addrp,
  93         __in            size_t size,
  94         __in            unsigned int n,
  95         __in            unsigned int completed,
  96         __in            unsigned int added);
  97
  98 static                  void
  99 siena_rx_qpush(
 100         __in            efx_rxq_t *erp,
 101         __in            unsigned int added,
 102         __inout         unsigned int *pushedp);
 103
 104 static  __checkReturn   efx_rc_t
 105 siena_rx_qflush(
 106         __in            efx_rxq_t *erp);
 107
 108 static                  void
 109 siena_rx_qenable(
 110         __in            efx_rxq_t *erp);
 111
 112 static  __checkReturn   efx_rc_t
 113 siena_rx_qcreate(
 114         __in            efx_nic_t *enp,
 115         __in            unsigned int index,
 116         __in            unsigned int label,
 117         __in            efx_rxq_type_t type,
 118         __in            efsys_mem_t *esmp,
 119         __in            size_t n,
 120         __in            uint32_t id,
 121         __in            efx_evq_t *eep,
 122         __in            efx_rxq_t *erp);
 123
 124 static                  void
 125 siena_rx_qdestroy(
 126         __in            efx_rxq_t *erp);
 127
 128 #endif /* EFSYS_OPT_SIENA */
 129
 130
 131 #if EFSYS_OPT_SIENA
 132 static const efx_rx_ops_t __efx_rx_siena_ops = {
 133         siena_rx_init,                          /* erxo_init */
 134         siena_rx_fini,                          /* erxo_fini */
 135 #if EFSYS_OPT_RX_SCATTER
 136         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 137 #endif
 138 #if EFSYS_OPT_RX_SCALE
 139         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 140         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 141         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 142         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 143 #endif
 144         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 145         siena_rx_qpost,                         /* erxo_qpost */
 146         siena_rx_qpush,                         /* erxo_qpush */
 147         siena_rx_qflush,                        /* erxo_qflush */
 148         siena_rx_qenable,                       /* erxo_qenable */
 149         siena_rx_qcreate,                       /* erxo_qcreate */
 150         siena_rx_qdestroy,                      /* erxo_qdestroy */
 151 };
 152 #endif  /* EFSYS_OPT_SIENA */
 153
 154 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 155 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 156         ef10_rx_init,                           /* erxo_init */
 157         ef10_rx_fini,                           /* erxo_fini */
 158 #if EFSYS_OPT_RX_SCATTER
 159         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 160 #endif
 161 #if EFSYS_OPT_RX_SCALE
 162         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 163         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 164         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 165         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 166 #endif
 167         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 168         ef10_rx_qpost,                          /* erxo_qpost */
 169         ef10_rx_qpush,                          /* erxo_qpush */
 170         ef10_rx_qflush,                         /* erxo_qflush */
 171         ef10_rx_qenable,                        /* erxo_qenable */
 172         ef10_rx_qcreate,                        /* erxo_qcreate */
 173         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 174 };
 175 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 176
 177
 178         __checkReturn   efx_rc_t
 179 efx_rx_init(
 180         __inout         efx_nic_t *enp)
 181 {
 182         const efx_rx_ops_t *erxop;
 183         efx_rc_t rc;
 184
 185         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 186         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 187
 188         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 189                 rc = EINVAL;
 190                 goto fail1;
 191         }
 192
 193         if (enp->en_mod_flags & EFX_MOD_RX) {
 194                 rc = EINVAL;
 195                 goto fail2;
 196         }
 197
 198         switch (enp->en_family) {
 199 #if EFSYS_OPT_SIENA
 200         case EFX_FAMILY_SIENA:
 201                 erxop = &__efx_rx_siena_ops;
 202                 break;
 203 #endif /* EFSYS_OPT_SIENA */
 204
 205 #if EFSYS_OPT_HUNTINGTON
 206         case EFX_FAMILY_HUNTINGTON:
 207                 erxop = &__efx_rx_ef10_ops;
 208                 break;
 209 #endif /* EFSYS_OPT_HUNTINGTON */
 210
 211 #if EFSYS_OPT_MEDFORD
 212         case EFX_FAMILY_MEDFORD:
 213                 erxop = &__efx_rx_ef10_ops;
 214                 break;
 215 #endif /* EFSYS_OPT_MEDFORD */
 216
 217         default:
 218                 EFSYS_ASSERT(0);
 219                 rc = ENOTSUP;
 220                 goto fail3;
 221         }
 222
 223         if ((rc = erxop->erxo_init(enp)) != 0)
 224                 goto fail4;
 225
 226         enp->en_erxop = erxop;
 227         enp->en_mod_flags |= EFX_MOD_RX;
 228         return (0);
 229
 230 fail4:
 231         EFSYS_PROBE(fail4);
 232 fail3:
 233         EFSYS_PROBE(fail3);
 234 fail2:
 235         EFSYS_PROBE(fail2);
 236 fail1:
 237         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 238
 239         enp->en_erxop = NULL;
 240         enp->en_mod_flags &= ~EFX_MOD_RX;
 241         return (rc);
 242 }
 243
 244                         void
 245 efx_rx_fini(
 246         __in            efx_nic_t *enp)
 247 {
 248         const efx_rx_ops_t *erxop = enp->en_erxop;
 249
 250         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 251         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 252         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 253         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 254
 255         erxop->erxo_fini(enp);
 256
 257         enp->en_erxop = NULL;
 258         enp->en_mod_flags &= ~EFX_MOD_RX;
 259 }
 260
 261 #if EFSYS_OPT_RX_SCATTER
 262         __checkReturn   efx_rc_t
 263 efx_rx_scatter_enable(
 264         __in            efx_nic_t *enp,
 265         __in            unsigned int buf_size)
 266 {
 267         const efx_rx_ops_t *erxop = enp->en_erxop;
 268         efx_rc_t rc;
 269
 270         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 271         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 272
 273         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 274                 goto fail1;
 275
 276         return (0);
 277
 278 fail1:
 279         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 280         return (rc);
 281 }
 282 #endif  /* EFSYS_OPT_RX_SCATTER */
 283
 284 #if EFSYS_OPT_RX_SCALE
 285         __checkReturn   efx_rc_t
 286 efx_rx_hash_support_get(
 287         __in            efx_nic_t *enp,
 288         __out           efx_rx_hash_support_t *supportp)
 289 {
 290         efx_rc_t rc;
 291
 292         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 293         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 294
 295         if (supportp == NULL) {
 296                 rc = EINVAL;
 297                 goto fail1;
 298         }
 299
 300         /* Report if resources are available to insert RX hash value */
 301         *supportp = enp->en_hash_support;
 302
 303         return (0);
 304
 305 fail1:
 306         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 307
 308         return (rc);
 309 }
 310
 311         __checkReturn   efx_rc_t
 312 efx_rx_scale_support_get(
 313         __in            efx_nic_t *enp,
 314         __out           efx_rx_scale_support_t *supportp)
 315 {
 316         efx_rc_t rc;
 317
 318         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 319         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 320
 321         if (supportp == NULL) {
 322                 rc = EINVAL;
 323                 goto fail1;
 324         }
 325
 326         /* Report if resources are available to support RSS */
 327         *supportp = enp->en_rss_support;
 328
 329         return (0);
 330
 331 fail1:
 332         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 333
 334         return (rc);
 335 }
 336
 337         __checkReturn   efx_rc_t
 338 efx_rx_scale_mode_set(
 339         __in            efx_nic_t *enp,
 340         __in            efx_rx_hash_alg_t alg,
 341         __in            efx_rx_hash_type_t type,
 342         __in            boolean_t insert)
 343 {
 344         const efx_rx_ops_t *erxop = enp->en_erxop;
 345         efx_rc_t rc;
 346
 347         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 348         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 349
 350         if (erxop->erxo_scale_mode_set != NULL) {
 351                 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
 352                             type, insert)) != 0)
 353                         goto fail1;
 354         }
 355
 356         return (0);
 357
 358 fail1:
 359         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 360         return (rc);
 361 }
 362 #endif  /* EFSYS_OPT_RX_SCALE */
 363
 364 #if EFSYS_OPT_RX_SCALE
 365         __checkReturn   efx_rc_t
 366 efx_rx_scale_key_set(
 367         __in            efx_nic_t *enp,
 368         __in_ecount(n)  uint8_t *key,
 369         __in            size_t n)
 370 {
 371         const efx_rx_ops_t *erxop = enp->en_erxop;
 372         efx_rc_t rc;
 373
 374         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 375         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 376
 377         if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
 378                 goto fail1;
 379
 380         return (0);
 381
 382 fail1:
 383         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 384
 385         return (rc);
 386 }
 387 #endif  /* EFSYS_OPT_RX_SCALE */
 388
 389 #if EFSYS_OPT_RX_SCALE
 390         __checkReturn   efx_rc_t
 391 efx_rx_scale_tbl_set(
 392         __in            efx_nic_t *enp,
 393         __in_ecount(n)  unsigned int *table,
 394         __in            size_t n)
 395 {
 396         const efx_rx_ops_t *erxop = enp->en_erxop;
 397         efx_rc_t rc;
 398
 399         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 400         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 401
 402         if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
 403                 goto fail1;
 404
 405         return (0);
 406
 407 fail1:
 408         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 409
 410         return (rc);
 411 }
 412 #endif  /* EFSYS_OPT_RX_SCALE */
 413
 414                         void
 415 efx_rx_qpost(
 416         __in            efx_rxq_t *erp,
 417         __in_ecount(n)  efsys_dma_addr_t *addrp,
 418         __in            size_t size,
 419         __in            unsigned int n,
 420         __in            unsigned int completed,
 421         __in            unsigned int added)
 422 {
 423         efx_nic_t *enp = erp->er_enp;
 424         const efx_rx_ops_t *erxop = enp->en_erxop;
 425
 426         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 427
 428         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 429 }
 430
 431                         void
 432 efx_rx_qpush(
 433         __in            efx_rxq_t *erp,
 434         __in            unsigned int added,
 435         __inout         unsigned int *pushedp)
 436 {
 437         efx_nic_t *enp = erp->er_enp;
 438         const efx_rx_ops_t *erxop = enp->en_erxop;
 439
 440         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 441
 442         erxop->erxo_qpush(erp, added, pushedp);
 443 }
 444
 445         __checkReturn   efx_rc_t
 446 efx_rx_qflush(
 447         __in            efx_rxq_t *erp)
 448 {
 449         efx_nic_t *enp = erp->er_enp;
 450         const efx_rx_ops_t *erxop = enp->en_erxop;
 451         efx_rc_t rc;
 452
 453         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 454
 455         if ((rc = erxop->erxo_qflush(erp)) != 0)
 456                 goto fail1;
 457
 458         return (0);
 459
 460 fail1:
 461         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 462
 463         return (rc);
 464 }
 465
 466                         void
 467 efx_rx_qenable(
 468         __in            efx_rxq_t *erp)
 469 {
 470         efx_nic_t *enp = erp->er_enp;
 471         const efx_rx_ops_t *erxop = enp->en_erxop;
 472
 473         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 474
 475         erxop->erxo_qenable(erp);
 476 }
 477
 478         __checkReturn   efx_rc_t
 479 efx_rx_qcreate(
 480         __in            efx_nic_t *enp,
 481         __in            unsigned int index,
 482         __in            unsigned int label,
 483         __in            efx_rxq_type_t type,
 484         __in            efsys_mem_t *esmp,
 485         __in            size_t n,
 486         __in            uint32_t id,
 487         __in            efx_evq_t *eep,
 488         __deref_out     efx_rxq_t **erpp)
 489 {
 490         const efx_rx_ops_t *erxop = enp->en_erxop;
 491         efx_rxq_t *erp;
 492         efx_rc_t rc;
 493
 494         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 495         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 496
 497         /* Allocate an RXQ object */
 498         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 499
 500         if (erp == NULL) {
 501                 rc = ENOMEM;
 502                 goto fail1;
 503         }
 504
 505         erp->er_magic = EFX_RXQ_MAGIC;
 506         erp->er_enp = enp;
 507         erp->er_index = index;
 508         erp->er_mask = n - 1;
 509         erp->er_esmp = esmp;
 510
 511         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 512             eep, erp)) != 0)
 513                 goto fail2;
 514
 515         enp->en_rx_qcount++;
 516         *erpp = erp;
 517
 518         return (0);
 519
 520 fail2:
 521         EFSYS_PROBE(fail2);
 522
 523         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 524 fail1:
 525         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 526
 527         return (rc);
 528 }
 529
 530                         void
 531 efx_rx_qdestroy(
 532         __in            efx_rxq_t *erp)
 533 {
 534         efx_nic_t *enp = erp->er_enp;
 535         const efx_rx_ops_t *erxop = enp->en_erxop;
 536
 537         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 538
 539         erxop->erxo_qdestroy(erp);
 540 }
 541
 542         __checkReturn   efx_rc_t
 543 efx_psuedo_hdr_pkt_length_get(
 544         __in            efx_nic_t *enp,
 545         __in            uint8_t *buffer,
 546         __out           uint16_t *lengthp)
 547 {
 548         const efx_rx_ops_t *erxop = enp->en_erxop;
 549
 550         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 551 }
 552
 553 #if EFSYS_OPT_RX_SCALE
 554         __checkReturn   uint32_t
 555 efx_psuedo_hdr_hash_get(
 556         __in            efx_nic_t *enp,
 557         __in            efx_rx_hash_alg_t func,
 558         __in            uint8_t *buffer)
 559 {
 560         const efx_rx_ops_t *erxop = enp->en_erxop;
 561
 562         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 563         return (erxop->erxo_prefix_hash(enp, func, buffer));
 564 }
 565 #endif  /* EFSYS_OPT_RX_SCALE */
 566
 567 #if EFSYS_OPT_SIENA
 568
 569 static  __checkReturn   efx_rc_t
 570 siena_rx_init(
 571         __in            efx_nic_t *enp)
 572 {
 573         efx_oword_t oword;
 574         unsigned int index;
 575
 576         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 577
 578         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 579         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 580         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 581         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 582         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 583         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 584         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 585
 586         /* Zero the RSS table */
 587         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 588             index++) {
 589                 EFX_ZERO_OWORD(oword);
 590                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 591                                     index, &oword, B_TRUE);
 592         }
 593
 594 #if EFSYS_OPT_RX_SCALE
 595         /* The RSS key and indirection table are writable. */
 596         enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
 597
 598         /* Hardware can insert RX hash with/without RSS */
 599         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 600 #endif  /* EFSYS_OPT_RX_SCALE */
 601
 602         return (0);
 603 }
 604
 605 #if EFSYS_OPT_RX_SCATTER
 606 static  __checkReturn   efx_rc_t
 607 siena_rx_scatter_enable(
 608         __in            efx_nic_t *enp,
 609         __in            unsigned int buf_size)
 610 {
 611         unsigned int nbuf32;
 612         efx_oword_t oword;
 613         efx_rc_t rc;
 614
 615         nbuf32 = buf_size / 32;
 616         if ((nbuf32 == 0) ||
 617             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 618             ((buf_size % 32) != 0)) {
 619                 rc = EINVAL;
 620                 goto fail1;
 621         }
 622
 623         if (enp->en_rx_qcount > 0) {
 624                 rc = EBUSY;
 625                 goto fail2;
 626         }
 627
 628         /* Set scatter buffer size */
 629         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 630         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 631         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 632
 633         /* Enable scatter for packets not matching a filter */
 634         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 635         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 636         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 637
 638         return (0);
 639
 640 fail2:
 641         EFSYS_PROBE(fail2);
 642 fail1:
 643         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 644
 645         return (rc);
 646 }
 647 #endif  /* EFSYS_OPT_RX_SCATTER */
 648
 649
 650 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 651         do {                                                            \
 652                 efx_oword_t oword;                                      \
 653                                                                         \
 654                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 655                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 656                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 657                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 658                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 659                     (_insert) ? 1 : 0);                                 \
 660                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 661                                                                         \
 662                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 663                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 664                             &oword);                                    \
 665                         EFX_SET_OWORD_FIELD(oword,                      \
 666                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 667                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 668                             &oword);                                    \
 669                 }                                                       \
 670                                                                         \
 671                 _NOTE(CONSTANTCONDITION)                                \
 672         } while (B_FALSE)
 673
 674 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 675         do {                                                            \
 676                 efx_oword_t oword;                                      \
 677                                                                         \
 678                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 679                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 680                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 681                     (_ip) ? 1 : 0);                                     \
 682                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 683                     (_tcp) ? 0 : 1);                                    \
 684                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 685                     (_insert) ? 1 : 0);                                 \
 686                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 687                                                                         \
 688                 _NOTE(CONSTANTCONDITION)                                \
 689         } while (B_FALSE)
 690
 691 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 692         do {                                                            \
 693                 efx_oword_t oword;                                      \
 694                                                                         \
 695                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 696                 EFX_SET_OWORD_FIELD(oword,                              \
 697                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 698                 EFX_SET_OWORD_FIELD(oword,                              \
 699                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 700                 EFX_SET_OWORD_FIELD(oword,                              \
 701                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 702                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 703                                                                         \
 704                 (_rc) = 0;                                              \
 705                                                                         \
 706                 _NOTE(CONSTANTCONDITION)                                \
 707         } while (B_FALSE)
 708
 709
 710 #if EFSYS_OPT_RX_SCALE
 711
 712 static  __checkReturn   efx_rc_t
 713 siena_rx_scale_mode_set(
 714         __in            efx_nic_t *enp,
 715         __in            efx_rx_hash_alg_t alg,
 716         __in            efx_rx_hash_type_t type,
 717         __in            boolean_t insert)
 718 {
 719         efx_rc_t rc;
 720
 721         switch (alg) {
 722         case EFX_RX_HASHALG_LFSR:
 723                 EFX_RX_LFSR_HASH(enp, insert);
 724                 break;
 725
 726         case EFX_RX_HASHALG_TOEPLITZ:
 727                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 728                     type & (1 << EFX_RX_HASH_IPV4),
 729                     type & (1 << EFX_RX_HASH_TCPIPV4));
 730
 731                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 732                     type & (1 << EFX_RX_HASH_IPV6),
 733                     type & (1 << EFX_RX_HASH_TCPIPV6),
 734                     rc);
 735                 if (rc != 0)
 736                         goto fail1;
 737
 738                 break;
 739
 740         default:
 741                 rc = EINVAL;
 742                 goto fail2;
 743         }
 744
 745         return (0);
 746
 747 fail2:
 748         EFSYS_PROBE(fail2);
 749 fail1:
 750         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 751
 752         EFX_RX_LFSR_HASH(enp, B_FALSE);
 753
 754         return (rc);
 755 }
 756 #endif
 757
 758 #if EFSYS_OPT_RX_SCALE
 759 static  __checkReturn   efx_rc_t
 760 siena_rx_scale_key_set(
 761         __in            efx_nic_t *enp,
 762         __in_ecount(n)  uint8_t *key,
 763         __in            size_t n)
 764 {
 765         efx_oword_t oword;
 766         unsigned int byte;
 767         unsigned int offset;
 768         efx_rc_t rc;
 769
 770         byte = 0;
 771
 772         /* Write Toeplitz IPv4 hash key */
 773         EFX_ZERO_OWORD(oword);
 774         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 775             offset > 0 && byte < n;
 776             --offset)
 777                 oword.eo_u8[offset - 1] = key[byte++];
 778
 779         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 780
 781         byte = 0;
 782
 783         /* Verify Toeplitz IPv4 hash key */
 784         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 785         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 786             offset > 0 && byte < n;
 787             --offset) {
 788                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 789                         rc = EFAULT;
 790                         goto fail1;
 791                 }
 792         }
 793
 794         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 795                 goto done;
 796
 797         byte = 0;
 798
 799         /* Write Toeplitz IPv6 hash key 3 */
 800         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 801         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 802             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 803             offset > 0 && byte < n;
 804             --offset)
 805                 oword.eo_u8[offset - 1] = key[byte++];
 806
 807         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 808
 809         /* Write Toeplitz IPv6 hash key 2 */
 810         EFX_ZERO_OWORD(oword);
 811         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 812             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 813             offset > 0 && byte < n;
 814             --offset)
 815                 oword.eo_u8[offset - 1] = key[byte++];
 816
 817         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 818
 819         /* Write Toeplitz IPv6 hash key 1 */
 820         EFX_ZERO_OWORD(oword);
 821         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 822             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 823             offset > 0 && byte < n;
 824             --offset)
 825                 oword.eo_u8[offset - 1] = key[byte++];
 826
 827         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 828
 829         byte = 0;
 830
 831         /* Verify Toeplitz IPv6 hash key 3 */
 832         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 833         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 834             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 835             offset > 0 && byte < n;
 836             --offset) {
 837                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 838                         rc = EFAULT;
 839                         goto fail2;
 840                 }
 841         }
 842
 843         /* Verify Toeplitz IPv6 hash key 2 */
 844         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 845         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 846             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 847             offset > 0 && byte < n;
 848             --offset) {
 849                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 850                         rc = EFAULT;
 851                         goto fail3;
 852                 }
 853         }
 854
 855         /* Verify Toeplitz IPv6 hash key 1 */
 856         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 857         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 858             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 859             offset > 0 && byte < n;
 860             --offset) {
 861                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 862                         rc = EFAULT;
 863                         goto fail4;
 864                 }
 865         }
 866
 867 done:
 868         return (0);
 869
 870 fail4:
 871         EFSYS_PROBE(fail4);
 872 fail3:
 873         EFSYS_PROBE(fail3);
 874 fail2:
 875         EFSYS_PROBE(fail2);
 876 fail1:
 877         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 878
 879         return (rc);
 880 }
 881 #endif
 882
 883 #if EFSYS_OPT_RX_SCALE
 884 static  __checkReturn   efx_rc_t
 885 siena_rx_scale_tbl_set(
 886         __in            efx_nic_t *enp,
 887         __in_ecount(n)  unsigned int *table,
 888         __in            size_t n)
 889 {
 890         efx_oword_t oword;
 891         int index;
 892         efx_rc_t rc;
 893
 894         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
 895         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
 896
 897         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
 898                 rc = EINVAL;
 899                 goto fail1;
 900         }
 901
 902         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
 903                 uint32_t byte;
 904
 905                 /* Calculate the entry to place in the table */
 906                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 907
 908                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
 909
 910                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
 911
 912                 /* Write the table */
 913                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 914                                     index, &oword, B_TRUE);
 915         }
 916
 917         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
 918                 uint32_t byte;
 919
 920                 /* Determine if we're starting a new batch */
 921                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 922
 923                 /* Read the table */
 924                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
 925                                     index, &oword, B_TRUE);
 926
 927                 /* Verify the entry */
 928                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
 929                         rc = EFAULT;
 930                         goto fail2;
 931                 }
 932         }
 933
 934         return (0);
 935
 936 fail2:
 937         EFSYS_PROBE(fail2);
 938 fail1:
 939         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 940
 941         return (rc);
 942 }
 943 #endif
 944
 945 /*
 946  * Falcon/Siena psuedo-header
 947  * --------------------------
 948  *
 949  * Receive packets are prefixed by an optional 16 byte pseudo-header.
 950  * The psuedo-header is a byte array of one of the forms:
 951  *
 952  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
 953  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
 954  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
 955  *
 956  * where:
 957  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
 958  *   LL.LL         LFSR hash     (16-bit big-endian)
 959  */
 960
 961 #if EFSYS_OPT_RX_SCALE
 962 static  __checkReturn   uint32_t
 963 siena_rx_prefix_hash(
 964         __in            efx_nic_t *enp,
 965         __in            efx_rx_hash_alg_t func,
 966         __in            uint8_t *buffer)
 967 {
 968         _NOTE(ARGUNUSED(enp))
 969
 970         switch (func) {
 971         case EFX_RX_HASHALG_TOEPLITZ:
 972                 return ((buffer[12] << 24) |
 973                     (buffer[13] << 16) |
 974                     (buffer[14] <<  8) |
 975                     buffer[15]);
 976
 977         case EFX_RX_HASHALG_LFSR:
 978                 return ((buffer[14] << 8) | buffer[15]);
 979
 980         default:
 981                 EFSYS_ASSERT(0);
 982                 return (0);
 983         }
 984 }
 985 #endif /* EFSYS_OPT_RX_SCALE */
 986
 987 static  __checkReturn   efx_rc_t
 988 siena_rx_prefix_pktlen(
 989         __in            efx_nic_t *enp,
 990         __in            uint8_t *buffer,
 991         __out           uint16_t *lengthp)
 992 {
 993         _NOTE(ARGUNUSED(enp, buffer, lengthp))
 994
 995         /* Not supported by Falcon/Siena hardware */
 996         EFSYS_ASSERT(0);
 997         return (ENOTSUP);
 998 }
 999
1000
1001 static                  void
1002 siena_rx_qpost(
1003         __in            efx_rxq_t *erp,
1004         __in_ecount(n)  efsys_dma_addr_t *addrp,
1005         __in            size_t size,
1006         __in            unsigned int n,
1007         __in            unsigned int completed,
1008         __in            unsigned int added)
1009 {
1010         efx_qword_t qword;
1011         unsigned int i;
1012         unsigned int offset;
1013         unsigned int id;
1014
1015         /* The client driver must not overfill the queue */
1016         EFSYS_ASSERT3U(added - completed + n, <=,
1017             EFX_RXQ_LIMIT(erp->er_mask + 1));
1018
1019         id = added & (erp->er_mask);
1020         for (i = 0; i < n; i++) {
1021                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1022                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1023                     size_t, size);
1024
1025                 EFX_POPULATE_QWORD_3(qword,
1026                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1027                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1028                     (uint32_t)(addrp[i] & 0xffffffff),
1029                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1030                     (uint32_t)(addrp[i] >> 32));
1031
1032                 offset = id * sizeof (efx_qword_t);
1033                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1034
1035                 id = (id + 1) & (erp->er_mask);
1036         }
1037 }
1038
1039 static                  void
1040 siena_rx_qpush(
1041         __in    efx_rxq_t *erp,
1042         __in    unsigned int added,
1043         __inout unsigned int *pushedp)
1044 {
1045         efx_nic_t *enp = erp->er_enp;
1046         unsigned int pushed = *pushedp;
1047         uint32_t wptr;
1048         efx_oword_t oword;
1049         efx_dword_t dword;
1050
1051         /* All descriptors are pushed */
1052         *pushedp = added;
1053
1054         /* Push the populated descriptors out */
1055         wptr = added & erp->er_mask;
1056
1057         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1058
1059         /* Only write the third DWORD */
1060         EFX_POPULATE_DWORD_1(dword,
1061             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1062
1063         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1064         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1065             wptr, pushed & erp->er_mask);
1066         EFSYS_PIO_WRITE_BARRIER();
1067         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1068                             erp->er_index, &dword, B_FALSE);
1069 }
1070
1071 static  __checkReturn   efx_rc_t
1072 siena_rx_qflush(
1073         __in    efx_rxq_t *erp)
1074 {
1075         efx_nic_t *enp = erp->er_enp;
1076         efx_oword_t oword;
1077         uint32_t label;
1078
1079         label = erp->er_index;
1080
1081         /* Flush the queue */
1082         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1083             FRF_AZ_RX_FLUSH_DESCQ, label);
1084         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1085
1086         return (0);
1087 }
1088
1089 static          void
1090 siena_rx_qenable(
1091         __in    efx_rxq_t *erp)
1092 {
1093         efx_nic_t *enp = erp->er_enp;
1094         efx_oword_t oword;
1095
1096         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1097
1098         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1099                             erp->er_index, &oword, B_TRUE);
1100
1101         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1102         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1103         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1104
1105         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1106                             erp->er_index, &oword, B_TRUE);
1107 }
1108
1109 static  __checkReturn   efx_rc_t
1110 siena_rx_qcreate(
1111         __in            efx_nic_t *enp,
1112         __in            unsigned int index,
1113         __in            unsigned int label,
1114         __in            efx_rxq_type_t type,
1115         __in            efsys_mem_t *esmp,
1116         __in            size_t n,
1117         __in            uint32_t id,
1118         __in            efx_evq_t *eep,
1119         __in            efx_rxq_t *erp)
1120 {
1121         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1122         efx_oword_t oword;
1123         uint32_t size;
1124         boolean_t jumbo;
1125         efx_rc_t rc;
1126
1127         _NOTE(ARGUNUSED(esmp))
1128
1129         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1130             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1131         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1132         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1133
1134         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1135         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1136
1137         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1138                 rc = EINVAL;
1139                 goto fail1;
1140         }
1141         if (index >= encp->enc_rxq_limit) {
1142                 rc = EINVAL;
1143                 goto fail2;
1144         }
1145         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1146             size++)
1147                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1148                         break;
1149         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1150                 rc = EINVAL;
1151                 goto fail3;
1152         }
1153
1154         switch (type) {
1155         case EFX_RXQ_TYPE_DEFAULT:
1156                 jumbo = B_FALSE;
1157                 break;
1158
1159 #if EFSYS_OPT_RX_SCATTER
1160         case EFX_RXQ_TYPE_SCATTER:
1161                 if (enp->en_family < EFX_FAMILY_SIENA) {
1162                         rc = EINVAL;
1163                         goto fail4;
1164                 }
1165                 jumbo = B_TRUE;
1166                 break;
1167 #endif  /* EFSYS_OPT_RX_SCATTER */
1168
1169         default:
1170                 rc = EINVAL;
1171                 goto fail4;
1172         }
1173
1174         /* Set up the new descriptor queue */
1175         EFX_POPULATE_OWORD_7(oword,
1176             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1177             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1178             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1179             FRF_AZ_RX_DESCQ_LABEL, label,
1180             FRF_AZ_RX_DESCQ_SIZE, size,
1181             FRF_AZ_RX_DESCQ_TYPE, 0,
1182             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1183
1184         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1185                             erp->er_index, &oword, B_TRUE);
1186
1187         return (0);
1188
1189 fail4:
1190         EFSYS_PROBE(fail4);
1191 fail3:
1192         EFSYS_PROBE(fail3);
1193 fail2:
1194         EFSYS_PROBE(fail2);
1195 fail1:
1196         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1197
1198         return (rc);
1199 }
1200
1201 static          void
1202 siena_rx_qdestroy(
1203         __in    efx_rxq_t *erp)
1204 {
1205         efx_nic_t *enp = erp->er_enp;
1206         efx_oword_t oword;
1207
1208         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1209         --enp->en_rx_qcount;
1210
1211         /* Purge descriptor queue */
1212         EFX_ZERO_OWORD(oword);
1213
1214         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1215                             erp->er_index, &oword, B_TRUE);
1216
1217         /* Free the RXQ object */
1218         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1219 }
1220
1221 static          void
1222 siena_rx_fini(
1223         __in    efx_nic_t *enp)
1224 {
1225         _NOTE(ARGUNUSED(enp))
1226 }
1227
1228 #endif /* EFSYS_OPT_SIENA */