sys/dev/sfxge/common/efx_rx.c

   1 /*-
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "efx.h"
  35 #include "efx_impl.h"
  36
  37
  38 #if EFSYS_OPT_SIENA
  39
  40 static  __checkReturn   efx_rc_t
  41 siena_rx_init(
  42         __in            efx_nic_t *enp);
  43
  44 static                  void
  45 siena_rx_fini(
  46         __in            efx_nic_t *enp);
  47
  48 #if EFSYS_OPT_RX_SCATTER
  49 static  __checkReturn   efx_rc_t
  50 siena_rx_scatter_enable(
  51         __in            efx_nic_t *enp,
  52         __in            unsigned int buf_size);
  53 #endif /* EFSYS_OPT_RX_SCATTER */
  54
  55 #if EFSYS_OPT_RX_SCALE
  56 static  __checkReturn   efx_rc_t
  57 siena_rx_scale_mode_set(
  58         __in            efx_nic_t *enp,
  59         __in            efx_rx_hash_alg_t alg,
  60         __in            efx_rx_hash_type_t type,
  61         __in            boolean_t insert);
  62
  63 static  __checkReturn   efx_rc_t
  64 siena_rx_scale_key_set(
  65         __in            efx_nic_t *enp,
  66         __in_ecount(n)  uint8_t *key,
  67         __in            size_t n);
  68
  69 static  __checkReturn   efx_rc_t
  70 siena_rx_scale_tbl_set(
  71         __in            efx_nic_t *enp,
  72         __in_ecount(n)  unsigned int *table,
  73         __in            size_t n);
  74
  75 static  __checkReturn   uint32_t
  76 siena_rx_prefix_hash(
  77         __in            efx_nic_t *enp,
  78         __in            efx_rx_hash_alg_t func,
  79         __in            uint8_t *buffer);
  80
  81 #endif /* EFSYS_OPT_RX_SCALE */
  82
  83 static  __checkReturn   efx_rc_t
  84 siena_rx_prefix_pktlen(
  85         __in            efx_nic_t *enp,
  86         __in            uint8_t *buffer,
  87         __out           uint16_t *lengthp);
  88
  89 static                  void
  90 siena_rx_qpost(
  91         __in            efx_rxq_t *erp,
  92         __in_ecount(n)  efsys_dma_addr_t *addrp,
  93         __in            size_t size,
  94         __in            unsigned int n,
  95         __in            unsigned int completed,
  96         __in            unsigned int added);
  97
  98 static                  void
  99 siena_rx_qpush(
 100         __in            efx_rxq_t *erp,
 101         __in            unsigned int added,
 102         __inout         unsigned int *pushedp);
 103
 104 static  __checkReturn   efx_rc_t
 105 siena_rx_qflush(
 106         __in            efx_rxq_t *erp);
 107
 108 static                  void
 109 siena_rx_qenable(
 110         __in            efx_rxq_t *erp);
 111
 112 static  __checkReturn   efx_rc_t
 113 siena_rx_qcreate(
 114         __in            efx_nic_t *enp,
 115         __in            unsigned int index,
 116         __in            unsigned int label,
 117         __in            efx_rxq_type_t type,
 118         __in            efsys_mem_t *esmp,
 119         __in            size_t n,
 120         __in            uint32_t id,
 121         __in            efx_evq_t *eep,
 122         __in            efx_rxq_t *erp);
 123
 124 static                  void
 125 siena_rx_qdestroy(
 126         __in            efx_rxq_t *erp);
 127
 128 #endif /* EFSYS_OPT_SIENA */
 129
 130
 131 #if EFSYS_OPT_SIENA
 132 static const efx_rx_ops_t __efx_rx_siena_ops = {
 133         siena_rx_init,                          /* erxo_init */
 134         siena_rx_fini,                          /* erxo_fini */
 135 #if EFSYS_OPT_RX_SCATTER
 136         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 137 #endif
 138 #if EFSYS_OPT_RX_SCALE
 139         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 140         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 141         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 142         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 143 #endif
 144         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 145         siena_rx_qpost,                         /* erxo_qpost */
 146         siena_rx_qpush,                         /* erxo_qpush */
 147         siena_rx_qflush,                        /* erxo_qflush */
 148         siena_rx_qenable,                       /* erxo_qenable */
 149         siena_rx_qcreate,                       /* erxo_qcreate */
 150         siena_rx_qdestroy,                      /* erxo_qdestroy */
 151 };
 152 #endif  /* EFSYS_OPT_SIENA */
 153
 154 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 155 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 156         ef10_rx_init,                           /* erxo_init */
 157         ef10_rx_fini,                           /* erxo_fini */
 158 #if EFSYS_OPT_RX_SCATTER
 159         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 160 #endif
 161 #if EFSYS_OPT_RX_SCALE
 162         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 163         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 164         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 165         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 166 #endif
 167         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 168         ef10_rx_qpost,                          /* erxo_qpost */
 169         ef10_rx_qpush,                          /* erxo_qpush */
 170         ef10_rx_qflush,                         /* erxo_qflush */
 171         ef10_rx_qenable,                        /* erxo_qenable */
 172         ef10_rx_qcreate,                        /* erxo_qcreate */
 173         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 174 };
 175 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 176
 177
 178         __checkReturn   efx_rc_t
 179 efx_rx_init(
 180         __inout         efx_nic_t *enp)
 181 {
 182         const efx_rx_ops_t *erxop;
 183         efx_rc_t rc;
 184
 185         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 186         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 187
 188         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 189                 rc = EINVAL;
 190                 goto fail1;
 191         }
 192
 193         if (enp->en_mod_flags & EFX_MOD_RX) {
 194                 rc = EINVAL;
 195                 goto fail2;
 196         }
 197
 198         switch (enp->en_family) {
 199 #if EFSYS_OPT_SIENA
 200         case EFX_FAMILY_SIENA:
 201                 erxop = &__efx_rx_siena_ops;
 202                 break;
 203 #endif /* EFSYS_OPT_SIENA */
 204
 205 #if EFSYS_OPT_HUNTINGTON
 206         case EFX_FAMILY_HUNTINGTON:
 207                 erxop = &__efx_rx_ef10_ops;
 208                 break;
 209 #endif /* EFSYS_OPT_HUNTINGTON */
 210
 211 #if EFSYS_OPT_MEDFORD
 212         case EFX_FAMILY_MEDFORD:
 213                 erxop = &__efx_rx_ef10_ops;
 214                 break;
 215 #endif /* EFSYS_OPT_MEDFORD */
 216
 217         default:
 218                 EFSYS_ASSERT(0);
 219                 rc = ENOTSUP;
 220                 goto fail3;
 221         }
 222
 223         if ((rc = erxop->erxo_init(enp)) != 0)
 224                 goto fail4;
 225
 226         enp->en_erxop = erxop;
 227         enp->en_mod_flags |= EFX_MOD_RX;
 228         return (0);
 229
 230 fail4:
 231         EFSYS_PROBE(fail4);
 232 fail3:
 233         EFSYS_PROBE(fail3);
 234 fail2:
 235         EFSYS_PROBE(fail2);
 236 fail1:
 237         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 238
 239         enp->en_erxop = NULL;
 240         enp->en_mod_flags &= ~EFX_MOD_RX;
 241         return (rc);
 242 }
 243
 244                         void
 245 efx_rx_fini(
 246         __in            efx_nic_t *enp)
 247 {
 248         const efx_rx_ops_t *erxop = enp->en_erxop;
 249
 250         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 251         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 252         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 253         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 254
 255         erxop->erxo_fini(enp);
 256
 257         enp->en_erxop = NULL;
 258         enp->en_mod_flags &= ~EFX_MOD_RX;
 259 }
 260
 261 #if EFSYS_OPT_RX_SCATTER
 262         __checkReturn   efx_rc_t
 263 efx_rx_scatter_enable(
 264         __in            efx_nic_t *enp,
 265         __in            unsigned int buf_size)
 266 {
 267         const efx_rx_ops_t *erxop = enp->en_erxop;
 268         efx_rc_t rc;
 269
 270         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 271         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 272
 273         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 274                 goto fail1;
 275
 276         return (0);
 277
 278 fail1:
 279         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 280         return (rc);
 281 }
 282 #endif  /* EFSYS_OPT_RX_SCATTER */
 283
 284 #if EFSYS_OPT_RX_SCALE
 285         __checkReturn   efx_rc_t
 286 efx_rx_hash_support_get(
 287         __in            efx_nic_t *enp,
 288         __out           efx_rx_hash_support_t *supportp)
 289 {
 290         efx_rc_t rc;
 291
 292         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 293         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 294
 295         if (supportp == NULL) {
 296                 rc = EINVAL;
 297                 goto fail1;
 298         }
 299
 300         /* Report if resources are available to insert RX hash value */
 301         *supportp = enp->en_hash_support;
 302
 303         return (0);
 304
 305 fail1:
 306         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 307
 308         return (rc);
 309 }
 310
 311         __checkReturn   efx_rc_t
 312 efx_rx_scale_support_get(
 313         __in            efx_nic_t *enp,
 314         __out           efx_rx_scale_support_t *supportp)
 315 {
 316         efx_rc_t rc;
 317
 318         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 319         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 320
 321         if (supportp == NULL) {
 322                 rc = EINVAL;
 323                 goto fail1;
 324         }
 325
 326         /* Report if resources are available to support RSS */
 327         *supportp = enp->en_rss_support;
 328
 329         return (0);
 330
 331 fail1:
 332         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 333
 334         return (rc);
 335 }
 336
 337         __checkReturn   efx_rc_t
 338 efx_rx_scale_mode_set(
 339         __in            efx_nic_t *enp,
 340         __in            efx_rx_hash_alg_t alg,
 341         __in            efx_rx_hash_type_t type,
 342         __in            boolean_t insert)
 343 {
 344         const efx_rx_ops_t *erxop = enp->en_erxop;
 345         efx_rc_t rc;
 346
 347         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 348         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 349
 350         if (erxop->erxo_scale_mode_set != NULL) {
 351                 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
 352                             type, insert)) != 0)
 353                         goto fail1;
 354         }
 355
 356         return (0);
 357
 358 fail1:
 359         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 360         return (rc);
 361 }
 362 #endif  /* EFSYS_OPT_RX_SCALE */
 363
 364 #if EFSYS_OPT_RX_SCALE
 365         __checkReturn   efx_rc_t
 366 efx_rx_scale_key_set(
 367         __in            efx_nic_t *enp,
 368         __in_ecount(n)  uint8_t *key,
 369         __in            size_t n)
 370 {
 371         const efx_rx_ops_t *erxop = enp->en_erxop;
 372         efx_rc_t rc;
 373
 374         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 375         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 376
 377         if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
 378                 goto fail1;
 379
 380         return (0);
 381
 382 fail1:
 383         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 384
 385         return (rc);
 386 }
 387 #endif  /* EFSYS_OPT_RX_SCALE */
 388
 389 #if EFSYS_OPT_RX_SCALE
 390         __checkReturn   efx_rc_t
 391 efx_rx_scale_tbl_set(
 392         __in            efx_nic_t *enp,
 393         __in_ecount(n)  unsigned int *table,
 394         __in            size_t n)
 395 {
 396         const efx_rx_ops_t *erxop = enp->en_erxop;
 397         efx_rc_t rc;
 398
 399         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 400         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 401
 402         if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
 403                 goto fail1;
 404
 405         return (0);
 406
 407 fail1:
 408         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 409
 410         return (rc);
 411 }
 412 #endif  /* EFSYS_OPT_RX_SCALE */
 413
 414                         void
 415 efx_rx_qpost(
 416         __in            efx_rxq_t *erp,
 417         __in_ecount(n)  efsys_dma_addr_t *addrp,
 418         __in            size_t size,
 419         __in            unsigned int n,
 420         __in            unsigned int completed,
 421         __in            unsigned int added)
 422 {
 423         efx_nic_t *enp = erp->er_enp;
 424         const efx_rx_ops_t *erxop = enp->en_erxop;
 425
 426         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 427
 428         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 429 }
 430
 431                         void
 432 efx_rx_qpush(
 433         __in            efx_rxq_t *erp,
 434         __in            unsigned int added,
 435         __inout         unsigned int *pushedp)
 436 {
 437         efx_nic_t *enp = erp->er_enp;
 438         const efx_rx_ops_t *erxop = enp->en_erxop;
 439
 440         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 441
 442         erxop->erxo_qpush(erp, added, pushedp);
 443 }
 444
 445         __checkReturn   efx_rc_t
 446 efx_rx_qflush(
 447         __in            efx_rxq_t *erp)
 448 {
 449         efx_nic_t *enp = erp->er_enp;
 450         const efx_rx_ops_t *erxop = enp->en_erxop;
 451         efx_rc_t rc;
 452
 453         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 454
 455         if ((rc = erxop->erxo_qflush(erp)) != 0)
 456                 goto fail1;
 457
 458         return (0);
 459
 460 fail1:
 461         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 462
 463         return (rc);
 464 }
 465
 466                         void
 467 efx_rx_qenable(
 468         __in            efx_rxq_t *erp)
 469 {
 470         efx_nic_t *enp = erp->er_enp;
 471         const efx_rx_ops_t *erxop = enp->en_erxop;
 472
 473         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 474
 475         erxop->erxo_qenable(erp);
 476 }
 477
 478         __checkReturn   efx_rc_t
 479 efx_rx_qcreate(
 480         __in            efx_nic_t *enp,
 481         __in            unsigned int index,
 482         __in            unsigned int label,
 483         __in            efx_rxq_type_t type,
 484         __in            efsys_mem_t *esmp,
 485         __in            size_t n,
 486         __in            uint32_t id,
 487         __in            efx_evq_t *eep,
 488         __deref_out     efx_rxq_t **erpp)
 489 {
 490         const efx_rx_ops_t *erxop = enp->en_erxop;
 491         efx_rxq_t *erp;
 492         efx_rc_t rc;
 493
 494         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 495         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 496
 497         /* Allocate an RXQ object */
 498         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 499
 500         if (erp == NULL) {
 501                 rc = ENOMEM;
 502                 goto fail1;
 503         }
 504
 505         erp->er_magic = EFX_RXQ_MAGIC;
 506         erp->er_enp = enp;
 507         erp->er_index = index;
 508         erp->er_mask = n - 1;
 509         erp->er_esmp = esmp;
 510
 511         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 512             eep, erp)) != 0)
 513                 goto fail2;
 514
 515         enp->en_rx_qcount++;
 516         *erpp = erp;
 517
 518         return (0);
 519
 520 fail2:
 521         EFSYS_PROBE(fail2);
 522
 523         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 524 fail1:
 525         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 526
 527         return (rc);
 528 }
 529
 530                         void
 531 efx_rx_qdestroy(
 532         __in            efx_rxq_t *erp)
 533 {
 534         efx_nic_t *enp = erp->er_enp;
 535         const efx_rx_ops_t *erxop = enp->en_erxop;
 536
 537         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 538
 539         erxop->erxo_qdestroy(erp);
 540 }
 541
 542         __checkReturn   efx_rc_t
 543 efx_pseudo_hdr_pkt_length_get(
 544         __in            efx_rxq_t *erp,
 545         __in            uint8_t *buffer,
 546         __out           uint16_t *lengthp)
 547 {
 548         efx_nic_t *enp = erp->er_enp;
 549         const efx_rx_ops_t *erxop = enp->en_erxop;
 550
 551         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 552
 553         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 554 }
 555
 556 #if EFSYS_OPT_RX_SCALE
 557         __checkReturn   uint32_t
 558 efx_pseudo_hdr_hash_get(
 559         __in            efx_rxq_t *erp,
 560         __in            efx_rx_hash_alg_t func,
 561         __in            uint8_t *buffer)
 562 {
 563         efx_nic_t *enp = erp->er_enp;
 564         const efx_rx_ops_t *erxop = enp->en_erxop;
 565
 566         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 567
 568         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 569         return (erxop->erxo_prefix_hash(enp, func, buffer));
 570 }
 571 #endif  /* EFSYS_OPT_RX_SCALE */
 572
 573 #if EFSYS_OPT_SIENA
 574
 575 static  __checkReturn   efx_rc_t
 576 siena_rx_init(
 577         __in            efx_nic_t *enp)
 578 {
 579         efx_oword_t oword;
 580         unsigned int index;
 581
 582         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 583
 584         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 585         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 586         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 587         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 588         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 589         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 590         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 591
 592         /* Zero the RSS table */
 593         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 594             index++) {
 595                 EFX_ZERO_OWORD(oword);
 596                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 597                                     index, &oword, B_TRUE);
 598         }
 599
 600 #if EFSYS_OPT_RX_SCALE
 601         /* The RSS key and indirection table are writable. */
 602         enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
 603
 604         /* Hardware can insert RX hash with/without RSS */
 605         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 606 #endif  /* EFSYS_OPT_RX_SCALE */
 607
 608         return (0);
 609 }
 610
 611 #if EFSYS_OPT_RX_SCATTER
 612 static  __checkReturn   efx_rc_t
 613 siena_rx_scatter_enable(
 614         __in            efx_nic_t *enp,
 615         __in            unsigned int buf_size)
 616 {
 617         unsigned int nbuf32;
 618         efx_oword_t oword;
 619         efx_rc_t rc;
 620
 621         nbuf32 = buf_size / 32;
 622         if ((nbuf32 == 0) ||
 623             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 624             ((buf_size % 32) != 0)) {
 625                 rc = EINVAL;
 626                 goto fail1;
 627         }
 628
 629         if (enp->en_rx_qcount > 0) {
 630                 rc = EBUSY;
 631                 goto fail2;
 632         }
 633
 634         /* Set scatter buffer size */
 635         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 636         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 637         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 638
 639         /* Enable scatter for packets not matching a filter */
 640         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 641         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 642         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 643
 644         return (0);
 645
 646 fail2:
 647         EFSYS_PROBE(fail2);
 648 fail1:
 649         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 650
 651         return (rc);
 652 }
 653 #endif  /* EFSYS_OPT_RX_SCATTER */
 654
 655
 656 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 657         do {                                                            \
 658                 efx_oword_t oword;                                      \
 659                                                                         \
 660                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 661                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 662                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 663                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 664                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 665                     (_insert) ? 1 : 0);                                 \
 666                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 667                                                                         \
 668                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 669                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 670                             &oword);                                    \
 671                         EFX_SET_OWORD_FIELD(oword,                      \
 672                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 673                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 674                             &oword);                                    \
 675                 }                                                       \
 676                                                                         \
 677                 _NOTE(CONSTANTCONDITION)                                \
 678         } while (B_FALSE)
 679
 680 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 681         do {                                                            \
 682                 efx_oword_t oword;                                      \
 683                                                                         \
 684                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 685                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 686                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 687                     (_ip) ? 1 : 0);                                     \
 688                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 689                     (_tcp) ? 0 : 1);                                    \
 690                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 691                     (_insert) ? 1 : 0);                                 \
 692                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 693                                                                         \
 694                 _NOTE(CONSTANTCONDITION)                                \
 695         } while (B_FALSE)
 696
 697 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 698         do {                                                            \
 699                 efx_oword_t oword;                                      \
 700                                                                         \
 701                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 702                 EFX_SET_OWORD_FIELD(oword,                              \
 703                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 704                 EFX_SET_OWORD_FIELD(oword,                              \
 705                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 706                 EFX_SET_OWORD_FIELD(oword,                              \
 707                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 708                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 709                                                                         \
 710                 (_rc) = 0;                                              \
 711                                                                         \
 712                 _NOTE(CONSTANTCONDITION)                                \
 713         } while (B_FALSE)
 714
 715
 716 #if EFSYS_OPT_RX_SCALE
 717
 718 static  __checkReturn   efx_rc_t
 719 siena_rx_scale_mode_set(
 720         __in            efx_nic_t *enp,
 721         __in            efx_rx_hash_alg_t alg,
 722         __in            efx_rx_hash_type_t type,
 723         __in            boolean_t insert)
 724 {
 725         efx_rc_t rc;
 726
 727         switch (alg) {
 728         case EFX_RX_HASHALG_LFSR:
 729                 EFX_RX_LFSR_HASH(enp, insert);
 730                 break;
 731
 732         case EFX_RX_HASHALG_TOEPLITZ:
 733                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 734                     type & EFX_RX_HASH_IPV4,
 735                     type & EFX_RX_HASH_TCPIPV4);
 736
 737                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 738                     type & EFX_RX_HASH_IPV6,
 739                     type & EFX_RX_HASH_TCPIPV6,
 740                     rc);
 741                 if (rc != 0)
 742                         goto fail1;
 743
 744                 break;
 745
 746         default:
 747                 rc = EINVAL;
 748                 goto fail2;
 749         }
 750
 751         return (0);
 752
 753 fail2:
 754         EFSYS_PROBE(fail2);
 755 fail1:
 756         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 757
 758         EFX_RX_LFSR_HASH(enp, B_FALSE);
 759
 760         return (rc);
 761 }
 762 #endif
 763
 764 #if EFSYS_OPT_RX_SCALE
 765 static  __checkReturn   efx_rc_t
 766 siena_rx_scale_key_set(
 767         __in            efx_nic_t *enp,
 768         __in_ecount(n)  uint8_t *key,
 769         __in            size_t n)
 770 {
 771         efx_oword_t oword;
 772         unsigned int byte;
 773         unsigned int offset;
 774         efx_rc_t rc;
 775
 776         byte = 0;
 777
 778         /* Write Toeplitz IPv4 hash key */
 779         EFX_ZERO_OWORD(oword);
 780         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 781             offset > 0 && byte < n;
 782             --offset)
 783                 oword.eo_u8[offset - 1] = key[byte++];
 784
 785         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 786
 787         byte = 0;
 788
 789         /* Verify Toeplitz IPv4 hash key */
 790         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 791         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 792             offset > 0 && byte < n;
 793             --offset) {
 794                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 795                         rc = EFAULT;
 796                         goto fail1;
 797                 }
 798         }
 799
 800         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 801                 goto done;
 802
 803         byte = 0;
 804
 805         /* Write Toeplitz IPv6 hash key 3 */
 806         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 807         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 808             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 809             offset > 0 && byte < n;
 810             --offset)
 811                 oword.eo_u8[offset - 1] = key[byte++];
 812
 813         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 814
 815         /* Write Toeplitz IPv6 hash key 2 */
 816         EFX_ZERO_OWORD(oword);
 817         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 818             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 819             offset > 0 && byte < n;
 820             --offset)
 821                 oword.eo_u8[offset - 1] = key[byte++];
 822
 823         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 824
 825         /* Write Toeplitz IPv6 hash key 1 */
 826         EFX_ZERO_OWORD(oword);
 827         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 828             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 829             offset > 0 && byte < n;
 830             --offset)
 831                 oword.eo_u8[offset - 1] = key[byte++];
 832
 833         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 834
 835         byte = 0;
 836
 837         /* Verify Toeplitz IPv6 hash key 3 */
 838         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 839         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 840             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 841             offset > 0 && byte < n;
 842             --offset) {
 843                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 844                         rc = EFAULT;
 845                         goto fail2;
 846                 }
 847         }
 848
 849         /* Verify Toeplitz IPv6 hash key 2 */
 850         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 851         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 852             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 853             offset > 0 && byte < n;
 854             --offset) {
 855                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 856                         rc = EFAULT;
 857                         goto fail3;
 858                 }
 859         }
 860
 861         /* Verify Toeplitz IPv6 hash key 1 */
 862         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 863         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 864             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 865             offset > 0 && byte < n;
 866             --offset) {
 867                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 868                         rc = EFAULT;
 869                         goto fail4;
 870                 }
 871         }
 872
 873 done:
 874         return (0);
 875
 876 fail4:
 877         EFSYS_PROBE(fail4);
 878 fail3:
 879         EFSYS_PROBE(fail3);
 880 fail2:
 881         EFSYS_PROBE(fail2);
 882 fail1:
 883         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 884
 885         return (rc);
 886 }
 887 #endif
 888
 889 #if EFSYS_OPT_RX_SCALE
 890 static  __checkReturn   efx_rc_t
 891 siena_rx_scale_tbl_set(
 892         __in            efx_nic_t *enp,
 893         __in_ecount(n)  unsigned int *table,
 894         __in            size_t n)
 895 {
 896         efx_oword_t oword;
 897         int index;
 898         efx_rc_t rc;
 899
 900         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
 901         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
 902
 903         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
 904                 rc = EINVAL;
 905                 goto fail1;
 906         }
 907
 908         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
 909                 uint32_t byte;
 910
 911                 /* Calculate the entry to place in the table */
 912                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 913
 914                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
 915
 916                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
 917
 918                 /* Write the table */
 919                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 920                                     index, &oword, B_TRUE);
 921         }
 922
 923         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
 924                 uint32_t byte;
 925
 926                 /* Determine if we're starting a new batch */
 927                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 928
 929                 /* Read the table */
 930                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
 931                                     index, &oword, B_TRUE);
 932
 933                 /* Verify the entry */
 934                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
 935                         rc = EFAULT;
 936                         goto fail2;
 937                 }
 938         }
 939
 940         return (0);
 941
 942 fail2:
 943         EFSYS_PROBE(fail2);
 944 fail1:
 945         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 946
 947         return (rc);
 948 }
 949 #endif
 950
 951 /*
 952  * Falcon/Siena pseudo-header
 953  * --------------------------
 954  *
 955  * Receive packets are prefixed by an optional 16 byte pseudo-header.
 956  * The pseudo-header is a byte array of one of the forms:
 957  *
 958  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
 959  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
 960  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
 961  *
 962  * where:
 963  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
 964  *   LL.LL         LFSR hash     (16-bit big-endian)
 965  */
 966
 967 #if EFSYS_OPT_RX_SCALE
 968 static  __checkReturn   uint32_t
 969 siena_rx_prefix_hash(
 970         __in            efx_nic_t *enp,
 971         __in            efx_rx_hash_alg_t func,
 972         __in            uint8_t *buffer)
 973 {
 974         _NOTE(ARGUNUSED(enp))
 975
 976         switch (func) {
 977         case EFX_RX_HASHALG_TOEPLITZ:
 978                 return ((buffer[12] << 24) |
 979                     (buffer[13] << 16) |
 980                     (buffer[14] <<  8) |
 981                     buffer[15]);
 982
 983         case EFX_RX_HASHALG_LFSR:
 984                 return ((buffer[14] << 8) | buffer[15]);
 985
 986         default:
 987                 EFSYS_ASSERT(0);
 988                 return (0);
 989         }
 990 }
 991 #endif /* EFSYS_OPT_RX_SCALE */
 992
 993 static  __checkReturn   efx_rc_t
 994 siena_rx_prefix_pktlen(
 995         __in            efx_nic_t *enp,
 996         __in            uint8_t *buffer,
 997         __out           uint16_t *lengthp)
 998 {
 999         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1000
1001         /* Not supported by Falcon/Siena hardware */
1002         EFSYS_ASSERT(0);
1003         return (ENOTSUP);
1004 }
1005
1006
1007 static                  void
1008 siena_rx_qpost(
1009         __in            efx_rxq_t *erp,
1010         __in_ecount(n)  efsys_dma_addr_t *addrp,
1011         __in            size_t size,
1012         __in            unsigned int n,
1013         __in            unsigned int completed,
1014         __in            unsigned int added)
1015 {
1016         efx_qword_t qword;
1017         unsigned int i;
1018         unsigned int offset;
1019         unsigned int id;
1020
1021         /* The client driver must not overfill the queue */
1022         EFSYS_ASSERT3U(added - completed + n, <=,
1023             EFX_RXQ_LIMIT(erp->er_mask + 1));
1024
1025         id = added & (erp->er_mask);
1026         for (i = 0; i < n; i++) {
1027                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1028                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1029                     size_t, size);
1030
1031                 EFX_POPULATE_QWORD_3(qword,
1032                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1033                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1034                     (uint32_t)(addrp[i] & 0xffffffff),
1035                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1036                     (uint32_t)(addrp[i] >> 32));
1037
1038                 offset = id * sizeof (efx_qword_t);
1039                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1040
1041                 id = (id + 1) & (erp->er_mask);
1042         }
1043 }
1044
1045 static                  void
1046 siena_rx_qpush(
1047         __in    efx_rxq_t *erp,
1048         __in    unsigned int added,
1049         __inout unsigned int *pushedp)
1050 {
1051         efx_nic_t *enp = erp->er_enp;
1052         unsigned int pushed = *pushedp;
1053         uint32_t wptr;
1054         efx_oword_t oword;
1055         efx_dword_t dword;
1056
1057         /* All descriptors are pushed */
1058         *pushedp = added;
1059
1060         /* Push the populated descriptors out */
1061         wptr = added & erp->er_mask;
1062
1063         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1064
1065         /* Only write the third DWORD */
1066         EFX_POPULATE_DWORD_1(dword,
1067             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1068
1069         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1070         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1071             wptr, pushed & erp->er_mask);
1072         EFSYS_PIO_WRITE_BARRIER();
1073         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1074                             erp->er_index, &dword, B_FALSE);
1075 }
1076
1077 static  __checkReturn   efx_rc_t
1078 siena_rx_qflush(
1079         __in    efx_rxq_t *erp)
1080 {
1081         efx_nic_t *enp = erp->er_enp;
1082         efx_oword_t oword;
1083         uint32_t label;
1084
1085         label = erp->er_index;
1086
1087         /* Flush the queue */
1088         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1089             FRF_AZ_RX_FLUSH_DESCQ, label);
1090         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1091
1092         return (0);
1093 }
1094
1095 static          void
1096 siena_rx_qenable(
1097         __in    efx_rxq_t *erp)
1098 {
1099         efx_nic_t *enp = erp->er_enp;
1100         efx_oword_t oword;
1101
1102         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1103
1104         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1105                             erp->er_index, &oword, B_TRUE);
1106
1107         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1108         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1109         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1110
1111         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1112                             erp->er_index, &oword, B_TRUE);
1113 }
1114
1115 static  __checkReturn   efx_rc_t
1116 siena_rx_qcreate(
1117         __in            efx_nic_t *enp,
1118         __in            unsigned int index,
1119         __in            unsigned int label,
1120         __in            efx_rxq_type_t type,
1121         __in            efsys_mem_t *esmp,
1122         __in            size_t n,
1123         __in            uint32_t id,
1124         __in            efx_evq_t *eep,
1125         __in            efx_rxq_t *erp)
1126 {
1127         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1128         efx_oword_t oword;
1129         uint32_t size;
1130         boolean_t jumbo;
1131         efx_rc_t rc;
1132
1133         _NOTE(ARGUNUSED(esmp))
1134
1135         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1136             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1137         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1138         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1139
1140         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1141         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1142
1143         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1144                 rc = EINVAL;
1145                 goto fail1;
1146         }
1147         if (index >= encp->enc_rxq_limit) {
1148                 rc = EINVAL;
1149                 goto fail2;
1150         }
1151         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1152             size++)
1153                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1154                         break;
1155         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1156                 rc = EINVAL;
1157                 goto fail3;
1158         }
1159
1160         switch (type) {
1161         case EFX_RXQ_TYPE_DEFAULT:
1162                 jumbo = B_FALSE;
1163                 break;
1164
1165 #if EFSYS_OPT_RX_SCATTER
1166         case EFX_RXQ_TYPE_SCATTER:
1167                 if (enp->en_family < EFX_FAMILY_SIENA) {
1168                         rc = EINVAL;
1169                         goto fail4;
1170                 }
1171                 jumbo = B_TRUE;
1172                 break;
1173 #endif  /* EFSYS_OPT_RX_SCATTER */
1174
1175         default:
1176                 rc = EINVAL;
1177                 goto fail4;
1178         }
1179
1180         /* Set up the new descriptor queue */
1181         EFX_POPULATE_OWORD_7(oword,
1182             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1183             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1184             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1185             FRF_AZ_RX_DESCQ_LABEL, label,
1186             FRF_AZ_RX_DESCQ_SIZE, size,
1187             FRF_AZ_RX_DESCQ_TYPE, 0,
1188             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1189
1190         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1191                             erp->er_index, &oword, B_TRUE);
1192
1193         return (0);
1194
1195 fail4:
1196         EFSYS_PROBE(fail4);
1197 fail3:
1198         EFSYS_PROBE(fail3);
1199 fail2:
1200         EFSYS_PROBE(fail2);
1201 fail1:
1202         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1203
1204         return (rc);
1205 }
1206
1207 static          void
1208 siena_rx_qdestroy(
1209         __in    efx_rxq_t *erp)
1210 {
1211         efx_nic_t *enp = erp->er_enp;
1212         efx_oword_t oword;
1213
1214         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1215         --enp->en_rx_qcount;
1216
1217         /* Purge descriptor queue */
1218         EFX_ZERO_OWORD(oword);
1219
1220         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1221                             erp->er_index, &oword, B_TRUE);
1222
1223         /* Free the RXQ object */
1224         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1225 }
1226
1227 static          void
1228 siena_rx_fini(
1229         __in    efx_nic_t *enp)
1230 {
1231         _NOTE(ARGUNUSED(enp))
1232 }
1233
1234 #endif /* EFSYS_OPT_SIENA */