sys/dev/sfxge/common/efx_rx.c

   1 /*-
   2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "efsys.h"
  35 #include "efx.h"
  36 #include "efx_types.h"
  37 #include "efx_regs.h"
  38 #include "efx_impl.h"
  39
  40
  41 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
  42
  43 static  __checkReturn   efx_rc_t
  44 falconsiena_rx_init(
  45         __in            efx_nic_t *enp);
  46
  47 static                  void
  48 falconsiena_rx_fini(
  49         __in            efx_nic_t *enp);
  50
  51 #if EFSYS_OPT_RX_SCATTER
  52 static  __checkReturn   efx_rc_t
  53 falconsiena_rx_scatter_enable(
  54         __in            efx_nic_t *enp,
  55         __in            unsigned int buf_size);
  56 #endif /* EFSYS_OPT_RX_SCATTER */
  57
  58 #if EFSYS_OPT_RX_SCALE
  59 static  __checkReturn   efx_rc_t
  60 falconsiena_rx_scale_mode_set(
  61         __in            efx_nic_t *enp,
  62         __in            efx_rx_hash_alg_t alg,
  63         __in            efx_rx_hash_type_t type,
  64         __in            boolean_t insert);
  65
  66 static  __checkReturn   efx_rc_t
  67 falconsiena_rx_scale_key_set(
  68         __in            efx_nic_t *enp,
  69         __in_ecount(n)  uint8_t *key,
  70         __in            size_t n);
  71
  72 static  __checkReturn   efx_rc_t
  73 falconsiena_rx_scale_tbl_set(
  74         __in            efx_nic_t *enp,
  75         __in_ecount(n)  unsigned int *table,
  76         __in            size_t n);
  77
  78 #endif /* EFSYS_OPT_RX_SCALE */
  79
  80 static                  void
  81 falconsiena_rx_qpost(
  82         __in            efx_rxq_t *erp,
  83         __in_ecount(n)  efsys_dma_addr_t *addrp,
  84         __in            size_t size,
  85         __in            unsigned int n,
  86         __in            unsigned int completed,
  87         __in            unsigned int added);
  88
  89 static                  void
  90 falconsiena_rx_qpush(
  91         __in            efx_rxq_t *erp,
  92         __in            unsigned int added,
  93         __inout         unsigned int *pushedp);
  94
  95 static  __checkReturn   efx_rc_t
  96 falconsiena_rx_qflush(
  97         __in            efx_rxq_t *erp);
  98
  99 static                  void
 100 falconsiena_rx_qenable(
 101         __in            efx_rxq_t *erp);
 102
 103 static  __checkReturn   efx_rc_t
 104 falconsiena_rx_qcreate(
 105         __in            efx_nic_t *enp,
 106         __in            unsigned int index,
 107         __in            unsigned int label,
 108         __in            efx_rxq_type_t type,
 109         __in            efsys_mem_t *esmp,
 110         __in            size_t n,
 111         __in            uint32_t id,
 112         __in            efx_evq_t *eep,
 113         __in            efx_rxq_t *erp);
 114
 115 static                  void
 116 falconsiena_rx_qdestroy(
 117         __in            efx_rxq_t *erp);
 118
 119 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
 120
 121
 122 #if EFSYS_OPT_FALCON
 123 static efx_rx_ops_t __efx_rx_falcon_ops = {
 124         falconsiena_rx_init,                    /* erxo_init */
 125         falconsiena_rx_fini,                    /* erxo_fini */
 126 #if EFSYS_OPT_RX_SCATTER
 127         falconsiena_rx_scatter_enable,          /* erxo_scatter_enable */
 128 #endif
 129 #if EFSYS_OPT_RX_SCALE
 130         falconsiena_rx_scale_mode_set,          /* erxo_scale_mode_set */
 131         falconsiena_rx_scale_key_set,           /* erxo_scale_key_set */
 132         falconsiena_rx_scale_tbl_set,           /* erxo_scale_tbl_set */
 133 #endif
 134         falconsiena_rx_qpost,                   /* erxo_qpost */
 135         falconsiena_rx_qpush,                   /* erxo_qpush */
 136         falconsiena_rx_qflush,                  /* erxo_qflush */
 137         falconsiena_rx_qenable,                 /* erxo_qenable */
 138         falconsiena_rx_qcreate,                 /* erxo_qcreate */
 139         falconsiena_rx_qdestroy,                /* erxo_qdestroy */
 140 };
 141 #endif  /* EFSYS_OPT_FALCON */
 142
 143 #if EFSYS_OPT_SIENA
 144 static efx_rx_ops_t __efx_rx_siena_ops = {
 145         falconsiena_rx_init,                    /* erxo_init */
 146         falconsiena_rx_fini,                    /* erxo_fini */
 147 #if EFSYS_OPT_RX_SCATTER
 148         falconsiena_rx_scatter_enable,          /* erxo_scatter_enable */
 149 #endif
 150 #if EFSYS_OPT_RX_SCALE
 151         falconsiena_rx_scale_mode_set,          /* erxo_scale_mode_set */
 152         falconsiena_rx_scale_key_set,           /* erxo_scale_key_set */
 153         falconsiena_rx_scale_tbl_set,           /* erxo_scale_tbl_set */
 154 #endif
 155         falconsiena_rx_qpost,                   /* erxo_qpost */
 156         falconsiena_rx_qpush,                   /* erxo_qpush */
 157         falconsiena_rx_qflush,                  /* erxo_qflush */
 158         falconsiena_rx_qenable,                 /* erxo_qenable */
 159         falconsiena_rx_qcreate,                 /* erxo_qcreate */
 160         falconsiena_rx_qdestroy,                /* erxo_qdestroy */
 161 };
 162 #endif  /* EFSYS_OPT_SIENA */
 163
 164 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 165 static efx_rx_ops_t __efx_rx_ef10_ops = {
 166         ef10_rx_init,                           /* erxo_init */
 167         ef10_rx_fini,                           /* erxo_fini */
 168 #if EFSYS_OPT_RX_SCATTER
 169         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 170 #endif
 171 #if EFSYS_OPT_RX_SCALE
 172         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 173         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 174         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 175 #endif
 176         ef10_rx_qpost,                          /* erxo_qpost */
 177         ef10_rx_qpush,                          /* erxo_qpush */
 178         ef10_rx_qflush,                         /* erxo_qflush */
 179         ef10_rx_qenable,                        /* erxo_qenable */
 180         ef10_rx_qcreate,                        /* erxo_qcreate */
 181         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 182 };
 183 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 184
 185
 186         __checkReturn   efx_rc_t
 187 efx_rx_init(
 188         __inout         efx_nic_t *enp)
 189 {
 190         efx_rx_ops_t *erxop;
 191         efx_rc_t rc;
 192
 193         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 194         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 195
 196         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 197                 rc = EINVAL;
 198                 goto fail1;
 199         }
 200
 201         if (enp->en_mod_flags & EFX_MOD_RX) {
 202                 rc = EINVAL;
 203                 goto fail2;
 204         }
 205
 206         switch (enp->en_family) {
 207 #if EFSYS_OPT_FALCON
 208         case EFX_FAMILY_FALCON:
 209                 erxop = (efx_rx_ops_t *)&__efx_rx_falcon_ops;
 210                 break;
 211 #endif /* EFSYS_OPT_FALCON */
 212
 213 #if EFSYS_OPT_SIENA
 214         case EFX_FAMILY_SIENA:
 215                 erxop = (efx_rx_ops_t *)&__efx_rx_siena_ops;
 216                 break;
 217 #endif /* EFSYS_OPT_SIENA */
 218
 219 #if EFSYS_OPT_HUNTINGTON
 220         case EFX_FAMILY_HUNTINGTON:
 221                 erxop = (efx_rx_ops_t *)&__efx_rx_ef10_ops;
 222                 break;
 223 #endif /* EFSYS_OPT_HUNTINGTON */
 224
 225 #if EFSYS_OPT_MEDFORD
 226         case EFX_FAMILY_MEDFORD:
 227                 erxop = (efx_rx_ops_t *)&__efx_rx_ef10_ops;
 228                 break;
 229 #endif /* EFSYS_OPT_MEDFORD */
 230
 231         default:
 232                 EFSYS_ASSERT(0);
 233                 rc = ENOTSUP;
 234                 goto fail3;
 235         }
 236
 237         if ((rc = erxop->erxo_init(enp)) != 0)
 238                 goto fail4;
 239
 240         enp->en_erxop = erxop;
 241         enp->en_mod_flags |= EFX_MOD_RX;
 242         return (0);
 243
 244 fail4:
 245         EFSYS_PROBE(fail4);
 246 fail3:
 247         EFSYS_PROBE(fail3);
 248 fail2:
 249         EFSYS_PROBE(fail2);
 250 fail1:
 251         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 252
 253         enp->en_erxop = NULL;
 254         enp->en_mod_flags &= ~EFX_MOD_RX;
 255         return (rc);
 256 }
 257
 258                         void
 259 efx_rx_fini(
 260         __in            efx_nic_t *enp)
 261 {
 262         efx_rx_ops_t *erxop = enp->en_erxop;
 263
 264         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 265         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 266         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 267         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 268
 269         erxop->erxo_fini(enp);
 270
 271         enp->en_erxop = NULL;
 272         enp->en_mod_flags &= ~EFX_MOD_RX;
 273 }
 274
 275 #if EFSYS_OPT_RX_SCATTER
 276         __checkReturn   efx_rc_t
 277 efx_rx_scatter_enable(
 278         __in            efx_nic_t *enp,
 279         __in            unsigned int buf_size)
 280 {
 281         efx_rx_ops_t *erxop = enp->en_erxop;
 282         efx_rc_t rc;
 283
 284         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 285         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 286
 287         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 288                 goto fail1;
 289
 290         return (0);
 291
 292 fail1:
 293         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 294         return (rc);
 295 }
 296 #endif  /* EFSYS_OPT_RX_SCATTER */
 297
 298 #if EFSYS_OPT_RX_SCALE
 299         __checkReturn   efx_rc_t
 300 efx_rx_hash_support_get(
 301         __in            efx_nic_t *enp,
 302         __out           efx_rx_hash_support_t *supportp)
 303 {
 304         efx_rc_t rc;
 305
 306         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 307         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 308
 309         if (supportp == NULL) {
 310                 rc = EINVAL;
 311                 goto fail1;
 312         }
 313
 314         /* Report if resources are available to insert RX hash value */
 315         *supportp = enp->en_hash_support;
 316
 317         return (0);
 318
 319 fail1:
 320         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 321
 322         return (rc);
 323 }
 324
 325         __checkReturn   efx_rc_t
 326 efx_rx_scale_support_get(
 327         __in            efx_nic_t *enp,
 328         __out           efx_rx_scale_support_t *supportp)
 329 {
 330         efx_rc_t rc;
 331
 332         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 333         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 334
 335         if (supportp == NULL) {
 336                 rc = EINVAL;
 337                 goto fail1;
 338         }
 339
 340         /* Report if resources are available to support RSS */
 341         *supportp = enp->en_rss_support;
 342
 343         return (0);
 344
 345 fail1:
 346         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 347
 348         return (rc);
 349 }
 350
 351         __checkReturn   efx_rc_t
 352 efx_rx_scale_mode_set(
 353         __in            efx_nic_t *enp,
 354         __in            efx_rx_hash_alg_t alg,
 355         __in            efx_rx_hash_type_t type,
 356         __in            boolean_t insert)
 357 {
 358         efx_rx_ops_t *erxop = enp->en_erxop;
 359         efx_rc_t rc;
 360
 361         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 362         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 363
 364         if (erxop->erxo_scale_mode_set != NULL) {
 365                 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
 366                             type, insert)) != 0)
 367                         goto fail1;
 368         }
 369
 370         return (0);
 371
 372 fail1:
 373         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 374         return (rc);
 375 }
 376 #endif  /* EFSYS_OPT_RX_SCALE */
 377
 378 #if EFSYS_OPT_RX_SCALE
 379         __checkReturn   efx_rc_t
 380 efx_rx_scale_key_set(
 381         __in            efx_nic_t *enp,
 382         __in_ecount(n)  uint8_t *key,
 383         __in            size_t n)
 384 {
 385         efx_rx_ops_t *erxop = enp->en_erxop;
 386         efx_rc_t rc;
 387
 388         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 389         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 390
 391         if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
 392                 goto fail1;
 393
 394         return (0);
 395
 396 fail1:
 397         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 398
 399         return (rc);
 400 }
 401 #endif  /* EFSYS_OPT_RX_SCALE */
 402
 403 #if EFSYS_OPT_RX_SCALE
 404         __checkReturn   efx_rc_t
 405 efx_rx_scale_tbl_set(
 406         __in            efx_nic_t *enp,
 407         __in_ecount(n)  unsigned int *table,
 408         __in            size_t n)
 409 {
 410         efx_rx_ops_t *erxop = enp->en_erxop;
 411         efx_rc_t rc;
 412
 413         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 414         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 415
 416         if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
 417                 goto fail1;
 418
 419         return (0);
 420
 421 fail1:
 422         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 423
 424         return (rc);
 425 }
 426 #endif  /* EFSYS_OPT_RX_SCALE */
 427
 428                         void
 429 efx_rx_qpost(
 430         __in            efx_rxq_t *erp,
 431         __in_ecount(n)  efsys_dma_addr_t *addrp,
 432         __in            size_t size,
 433         __in            unsigned int n,
 434         __in            unsigned int completed,
 435         __in            unsigned int added)
 436 {
 437         efx_nic_t *enp = erp->er_enp;
 438         efx_rx_ops_t *erxop = enp->en_erxop;
 439
 440         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 441
 442         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 443 }
 444
 445                         void
 446 efx_rx_qpush(
 447         __in            efx_rxq_t *erp,
 448         __in            unsigned int added,
 449         __inout         unsigned int *pushedp)
 450 {
 451         efx_nic_t *enp = erp->er_enp;
 452         efx_rx_ops_t *erxop = enp->en_erxop;
 453
 454         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 455
 456         erxop->erxo_qpush(erp, added, pushedp);
 457 }
 458
 459         __checkReturn   efx_rc_t
 460 efx_rx_qflush(
 461         __in            efx_rxq_t *erp)
 462 {
 463         efx_nic_t *enp = erp->er_enp;
 464         efx_rx_ops_t *erxop = enp->en_erxop;
 465         efx_rc_t rc;
 466
 467         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 468
 469         if ((rc = erxop->erxo_qflush(erp)) != 0)
 470                 goto fail1;
 471
 472         return (0);
 473
 474 fail1:
 475         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 476
 477         return (rc);
 478 }
 479
 480                         void
 481 efx_rx_qenable(
 482         __in            efx_rxq_t *erp)
 483 {
 484         efx_nic_t *enp = erp->er_enp;
 485         efx_rx_ops_t *erxop = enp->en_erxop;
 486
 487         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 488
 489         erxop->erxo_qenable(erp);
 490 }
 491
 492         __checkReturn   efx_rc_t
 493 efx_rx_qcreate(
 494         __in            efx_nic_t *enp,
 495         __in            unsigned int index,
 496         __in            unsigned int label,
 497         __in            efx_rxq_type_t type,
 498         __in            efsys_mem_t *esmp,
 499         __in            size_t n,
 500         __in            uint32_t id,
 501         __in            efx_evq_t *eep,
 502         __deref_out     efx_rxq_t **erpp)
 503 {
 504         efx_rx_ops_t *erxop = enp->en_erxop;
 505         efx_rxq_t *erp;
 506         efx_rc_t rc;
 507
 508         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 509         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 510
 511         /* Allocate an RXQ object */
 512         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 513
 514         if (erp == NULL) {
 515                 rc = ENOMEM;
 516                 goto fail1;
 517         }
 518
 519         erp->er_magic = EFX_RXQ_MAGIC;
 520         erp->er_enp = enp;
 521         erp->er_index = index;
 522         erp->er_mask = n - 1;
 523         erp->er_esmp = esmp;
 524
 525         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 526             eep, erp)) != 0)
 527                 goto fail2;
 528
 529         enp->en_rx_qcount++;
 530         *erpp = erp;
 531
 532         return (0);
 533
 534 fail2:
 535         EFSYS_PROBE(fail2);
 536
 537         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 538 fail1:
 539         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 540
 541         return (rc);
 542 }
 543
 544                         void
 545 efx_rx_qdestroy(
 546         __in            efx_rxq_t *erp)
 547 {
 548         efx_nic_t *enp = erp->er_enp;
 549         efx_rx_ops_t *erxop = enp->en_erxop;
 550
 551         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 552
 553         erxop->erxo_qdestroy(erp);
 554 }
 555
 556 /*
 557  * Psuedo-header info for Siena/Falcon.
 558  *
 559  * The psuedo-header is a byte array of one of the forms:
 560  *
 561  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
 562  * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.TT.TT.TT.TT
 563  * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.LL.LL
 564  *
 565  * where:
 566  *
 567  * TT.TT.TT.TT is a 32-bit Toeplitz hash
 568  * LL.LL is a 16-bit LFSR hash
 569  *
 570  * Hash values are in network (big-endian) byte order.
 571  *
 572  *
 573  * On EF10 the pseudo-header is laid out as:
 574  * (See also SF-109306-TC section 9)
 575  *
 576  * Toeplitz hash (32 bits, little-endian)
 577  * Out-of-band outer VLAN tag
 578  *     (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag)
 579  * Out-of-band inner VLAN tag
 580  *     (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag)
 581  * Packet length (16 bits, little-endian, may be 0)
 582  * MAC timestamp (32 bits, little-endian, may be 0)
 583  * VLAN tag
 584  *     (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag)
 585  * VLAN tag
 586  *     (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag)
 587  */
 588
 589         __checkReturn   efx_rc_t
 590 efx_psuedo_hdr_pkt_length_get(
 591         __in            efx_nic_t *enp,
 592         __in            uint8_t *buffer,
 593         __out           uint16_t *pkt_lengthp)
 594 {
 595         if (enp->en_family != EFX_FAMILY_HUNTINGTON &&
 596             enp->en_family != EFX_FAMILY_MEDFORD) {
 597                 EFSYS_ASSERT(0);
 598                 return (ENOTSUP);
 599         }
 600
 601         *pkt_lengthp = buffer[8] | (buffer[9] << 8);
 602
 603         return (0);
 604 }
 605
 606 #if EFSYS_OPT_RX_SCALE
 607
 608 uint32_t
 609 efx_psuedo_hdr_hash_get(
 610         __in            efx_nic_t *enp,
 611         __in            efx_rx_hash_alg_t func,
 612         __in            uint8_t *buffer)
 613 {
 614         if (func == EFX_RX_HASHALG_TOEPLITZ) {
 615                 switch (enp->en_family) {
 616                 case EFX_FAMILY_FALCON:
 617                 case EFX_FAMILY_SIENA:
 618                         return ((buffer[12] << 24) |
 619                             (buffer[13] << 16) |
 620                             (buffer[14] << 8) |
 621                             buffer[15]);
 622                 case EFX_FAMILY_HUNTINGTON:
 623                 case EFX_FAMILY_MEDFORD:
 624                         return (buffer[0] |
 625                             (buffer[1] << 8) |
 626                             (buffer[2] << 16) |
 627                             (buffer[3] << 24));
 628                 default:
 629                         EFSYS_ASSERT(0);
 630                         return (0);
 631                 }
 632         } else if (func == EFX_RX_HASHALG_LFSR) {
 633                 EFSYS_ASSERT(enp->en_family == EFX_FAMILY_FALCON ||
 634                     enp->en_family == EFX_FAMILY_SIENA);
 635                 return ((buffer[14] << 8) | buffer[15]);
 636         } else {
 637                 EFSYS_ASSERT(0);
 638                 return (0);
 639         }
 640 }
 641
 642 #endif  /* EFSYS_OPT_RX_SCALE */
 643
 644 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
 645
 646 static  __checkReturn   efx_rc_t
 647 falconsiena_rx_init(
 648         __in            efx_nic_t *enp)
 649 {
 650         efx_oword_t oword;
 651         unsigned int index;
 652
 653         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 654
 655         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 656         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 657         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 658         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 659         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 660         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 661         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 662
 663         /* Zero the RSS table */
 664         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 665             index++) {
 666                 EFX_ZERO_OWORD(oword);
 667                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 668                                     index, &oword, B_TRUE);
 669         }
 670
 671 #if EFSYS_OPT_RX_SCALE
 672         /* The RSS key and indirection table are writable. */
 673         enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
 674
 675         /* Hardware can insert RX hash with/without RSS */
 676         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 677 #endif  /* EFSYS_OPT_RX_SCALE */
 678
 679         return (0);
 680 }
 681
 682 #if EFSYS_OPT_RX_SCATTER
 683 static  __checkReturn   efx_rc_t
 684 falconsiena_rx_scatter_enable(
 685         __in            efx_nic_t *enp,
 686         __in            unsigned int buf_size)
 687 {
 688         unsigned int nbuf32;
 689         efx_oword_t oword;
 690         efx_rc_t rc;
 691
 692         nbuf32 = buf_size / 32;
 693         if ((nbuf32 == 0) ||
 694             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 695             ((buf_size % 32) != 0)) {
 696                 rc = EINVAL;
 697                 goto fail1;
 698         }
 699
 700         if (enp->en_rx_qcount > 0) {
 701                 rc = EBUSY;
 702                 goto fail2;
 703         }
 704
 705         /* Set scatter buffer size */
 706         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 707         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 708         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 709
 710         /* Enable scatter for packets not matching a filter */
 711         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 712         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 713         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 714
 715         return (0);
 716
 717 fail2:
 718         EFSYS_PROBE(fail2);
 719 fail1:
 720         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 721
 722         return (rc);
 723 }
 724 #endif  /* EFSYS_OPT_RX_SCATTER */
 725
 726
 727 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 728         do {                                                            \
 729                 efx_oword_t oword;                                      \
 730                                                                         \
 731                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 732                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 733                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 734                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 735                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 736                     (_insert) ? 1 : 0);                                 \
 737                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 738                                                                         \
 739                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 740                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 741                             &oword);                                    \
 742                         EFX_SET_OWORD_FIELD(oword,                      \
 743                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 744                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 745                             &oword);                                    \
 746                 }                                                       \
 747                                                                         \
 748                 _NOTE(CONSTANTCONDITION)                                \
 749         } while (B_FALSE)
 750
 751 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 752         do {                                                            \
 753                 efx_oword_t oword;                                      \
 754                                                                         \
 755                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 756                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 757                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 758                     (_ip) ? 1 : 0);                                     \
 759                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 760                     (_tcp) ? 0 : 1);                                    \
 761                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 762                     (_insert) ? 1 : 0);                                 \
 763                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 764                                                                         \
 765                 _NOTE(CONSTANTCONDITION)                                \
 766         } while (B_FALSE)
 767
 768 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 769         do {                                                            \
 770                 efx_oword_t oword;                                      \
 771                                                                         \
 772                 if ((_enp)->en_family == EFX_FAMILY_FALCON) {           \
 773                         (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0;        \
 774                         break;                                          \
 775                 }                                                       \
 776                                                                         \
 777                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 778                 EFX_SET_OWORD_FIELD(oword,                              \
 779                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 780                 EFX_SET_OWORD_FIELD(oword,                              \
 781                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 782                 EFX_SET_OWORD_FIELD(oword,                              \
 783                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 784                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 785                                                                         \
 786                 (_rc) = 0;                                              \
 787                                                                         \
 788                 _NOTE(CONSTANTCONDITION)                                \
 789         } while (B_FALSE)
 790
 791
 792 #if EFSYS_OPT_RX_SCALE
 793
 794 static  __checkReturn   efx_rc_t
 795 falconsiena_rx_scale_mode_set(
 796         __in            efx_nic_t *enp,
 797         __in            efx_rx_hash_alg_t alg,
 798         __in            efx_rx_hash_type_t type,
 799         __in            boolean_t insert)
 800 {
 801         efx_rc_t rc;
 802
 803         switch (alg) {
 804         case EFX_RX_HASHALG_LFSR:
 805                 EFX_RX_LFSR_HASH(enp, insert);
 806                 break;
 807
 808         case EFX_RX_HASHALG_TOEPLITZ:
 809                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 810                     type & (1 << EFX_RX_HASH_IPV4),
 811                     type & (1 << EFX_RX_HASH_TCPIPV4));
 812
 813                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 814                     type & (1 << EFX_RX_HASH_IPV6),
 815                     type & (1 << EFX_RX_HASH_TCPIPV6),
 816                     rc);
 817                 if (rc != 0)
 818                         goto fail1;
 819
 820                 break;
 821
 822         default:
 823                 rc = EINVAL;
 824                 goto fail2;
 825         }
 826
 827         return (0);
 828
 829 fail2:
 830         EFSYS_PROBE(fail2);
 831 fail1:
 832         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 833
 834         EFX_RX_LFSR_HASH(enp, B_FALSE);
 835
 836         return (rc);
 837 }
 838 #endif
 839
 840 #if EFSYS_OPT_RX_SCALE
 841 static  __checkReturn   efx_rc_t
 842 falconsiena_rx_scale_key_set(
 843         __in            efx_nic_t *enp,
 844         __in_ecount(n)  uint8_t *key,
 845         __in            size_t n)
 846 {
 847         efx_oword_t oword;
 848         unsigned int byte;
 849         unsigned int offset;
 850         efx_rc_t rc;
 851
 852         byte = 0;
 853
 854         /* Write Toeplitz IPv4 hash key */
 855         EFX_ZERO_OWORD(oword);
 856         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 857             offset > 0 && byte < n;
 858             --offset)
 859                 oword.eo_u8[offset - 1] = key[byte++];
 860
 861         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 862
 863         byte = 0;
 864
 865         /* Verify Toeplitz IPv4 hash key */
 866         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 867         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 868             offset > 0 && byte < n;
 869             --offset) {
 870                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 871                         rc = EFAULT;
 872                         goto fail1;
 873                 }
 874         }
 875
 876         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 877                 goto done;
 878
 879         EFSYS_ASSERT3U(enp->en_family, !=, EFX_FAMILY_FALCON);
 880
 881         byte = 0;
 882
 883         /* Write Toeplitz IPv6 hash key 3 */
 884         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 885         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 886             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 887             offset > 0 && byte < n;
 888             --offset)
 889                 oword.eo_u8[offset - 1] = key[byte++];
 890
 891         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 892
 893         /* Write Toeplitz IPv6 hash key 2 */
 894         EFX_ZERO_OWORD(oword);
 895         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 896             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 897             offset > 0 && byte < n;
 898             --offset)
 899                 oword.eo_u8[offset - 1] = key[byte++];
 900
 901         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 902
 903         /* Write Toeplitz IPv6 hash key 1 */
 904         EFX_ZERO_OWORD(oword);
 905         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 906             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 907             offset > 0 && byte < n;
 908             --offset)
 909                 oword.eo_u8[offset - 1] = key[byte++];
 910
 911         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 912
 913         byte = 0;
 914
 915         /* Verify Toeplitz IPv6 hash key 3 */
 916         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 917         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 918             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 919             offset > 0 && byte < n;
 920             --offset) {
 921                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 922                         rc = EFAULT;
 923                         goto fail2;
 924                 }
 925         }
 926
 927         /* Verify Toeplitz IPv6 hash key 2 */
 928         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 929         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 930             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 931             offset > 0 && byte < n;
 932             --offset) {
 933                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 934                         rc = EFAULT;
 935                         goto fail3;
 936                 }
 937         }
 938
 939         /* Verify Toeplitz IPv6 hash key 1 */
 940         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 941         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 942             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 943             offset > 0 && byte < n;
 944             --offset) {
 945                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 946                         rc = EFAULT;
 947                         goto fail4;
 948                 }
 949         }
 950
 951 done:
 952         return (0);
 953
 954 fail4:
 955         EFSYS_PROBE(fail4);
 956 fail3:
 957         EFSYS_PROBE(fail3);
 958 fail2:
 959         EFSYS_PROBE(fail2);
 960 fail1:
 961         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 962
 963         return (rc);
 964 }
 965 #endif
 966
 967 #if EFSYS_OPT_RX_SCALE
 968 static  __checkReturn   efx_rc_t
 969 falconsiena_rx_scale_tbl_set(
 970         __in            efx_nic_t *enp,
 971         __in_ecount(n)  unsigned int *table,
 972         __in            size_t n)
 973 {
 974         efx_oword_t oword;
 975         int index;
 976         efx_rc_t rc;
 977
 978         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
 979         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
 980
 981         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
 982                 rc = EINVAL;
 983                 goto fail1;
 984         }
 985
 986         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
 987                 uint32_t byte;
 988
 989                 /* Calculate the entry to place in the table */
 990                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 991
 992                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
 993
 994                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
 995
 996                 /* Write the table */
 997                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 998                                     index, &oword, B_TRUE);
 999         }
1000
1001         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1002                 uint32_t byte;
1003
1004                 /* Determine if we're starting a new batch */
1005                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1006
1007                 /* Read the table */
1008                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1009                                     index, &oword, B_TRUE);
1010
1011                 /* Verify the entry */
1012                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1013                         rc = EFAULT;
1014                         goto fail2;
1015                 }
1016         }
1017
1018         return (0);
1019
1020 fail2:
1021         EFSYS_PROBE(fail2);
1022 fail1:
1023         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1024
1025         return (rc);
1026 }
1027 #endif
1028
1029 static                  void
1030 falconsiena_rx_qpost(
1031         __in            efx_rxq_t *erp,
1032         __in_ecount(n)  efsys_dma_addr_t *addrp,
1033         __in            size_t size,
1034         __in            unsigned int n,
1035         __in            unsigned int completed,
1036         __in            unsigned int added)
1037 {
1038         efx_qword_t qword;
1039         unsigned int i;
1040         unsigned int offset;
1041         unsigned int id;
1042
1043         /* The client driver must not overfill the queue */
1044         EFSYS_ASSERT3U(added - completed + n, <=,
1045             EFX_RXQ_LIMIT(erp->er_mask + 1));
1046
1047         id = added & (erp->er_mask);
1048         for (i = 0; i < n; i++) {
1049                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1050                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1051                     size_t, size);
1052
1053                 EFX_POPULATE_QWORD_3(qword,
1054                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1055                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1056                     (uint32_t)(addrp[i] & 0xffffffff),
1057                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1058                     (uint32_t)(addrp[i] >> 32));
1059
1060                 offset = id * sizeof (efx_qword_t);
1061                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1062
1063                 id = (id + 1) & (erp->er_mask);
1064         }
1065 }
1066
1067 static                  void
1068 falconsiena_rx_qpush(
1069         __in    efx_rxq_t *erp,
1070         __in    unsigned int added,
1071         __inout unsigned int *pushedp)
1072 {
1073         efx_nic_t *enp = erp->er_enp;
1074         unsigned int pushed = *pushedp;
1075         uint32_t wptr;
1076         efx_oword_t oword;
1077         efx_dword_t dword;
1078
1079         /* All descriptors are pushed */
1080         *pushedp = added;
1081
1082         /* Push the populated descriptors out */
1083         wptr = added & erp->er_mask;
1084
1085         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1086
1087         /* Only write the third DWORD */
1088         EFX_POPULATE_DWORD_1(dword,
1089             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1090
1091         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1092         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1093             wptr, pushed & erp->er_mask);
1094         EFSYS_PIO_WRITE_BARRIER();
1095         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1096                             erp->er_index, &dword, B_FALSE);
1097 }
1098
1099 static  __checkReturn   efx_rc_t
1100 falconsiena_rx_qflush(
1101         __in    efx_rxq_t *erp)
1102 {
1103         efx_nic_t *enp = erp->er_enp;
1104         efx_oword_t oword;
1105         uint32_t label;
1106
1107         label = erp->er_index;
1108
1109         /* Flush the queue */
1110         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1111             FRF_AZ_RX_FLUSH_DESCQ, label);
1112         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1113
1114         return (0);
1115 }
1116
1117 static          void
1118 falconsiena_rx_qenable(
1119         __in    efx_rxq_t *erp)
1120 {
1121         efx_nic_t *enp = erp->er_enp;
1122         efx_oword_t oword;
1123
1124         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1125
1126         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1127                             erp->er_index, &oword, B_TRUE);
1128
1129         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1130         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1131         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1132
1133         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1134                             erp->er_index, &oword, B_TRUE);
1135 }
1136
1137 static  __checkReturn   efx_rc_t
1138 falconsiena_rx_qcreate(
1139         __in            efx_nic_t *enp,
1140         __in            unsigned int index,
1141         __in            unsigned int label,
1142         __in            efx_rxq_type_t type,
1143         __in            efsys_mem_t *esmp,
1144         __in            size_t n,
1145         __in            uint32_t id,
1146         __in            efx_evq_t *eep,
1147         __in            efx_rxq_t *erp)
1148 {
1149         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1150         efx_oword_t oword;
1151         uint32_t size;
1152         boolean_t jumbo;
1153         efx_rc_t rc;
1154
1155         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1156             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1157         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1158         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1159
1160         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1161         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1162
1163         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1164                 rc = EINVAL;
1165                 goto fail1;
1166         }
1167         if (index >= encp->enc_rxq_limit) {
1168                 rc = EINVAL;
1169                 goto fail2;
1170         }
1171         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1172             size++)
1173                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1174                         break;
1175         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1176                 rc = EINVAL;
1177                 goto fail3;
1178         }
1179
1180         switch (type) {
1181         case EFX_RXQ_TYPE_DEFAULT:
1182                 jumbo = B_FALSE;
1183                 break;
1184
1185 #if EFSYS_OPT_RX_SCATTER
1186         case EFX_RXQ_TYPE_SCATTER:
1187                 if (enp->en_family < EFX_FAMILY_SIENA) {
1188                         rc = EINVAL;
1189                         goto fail4;
1190                 }
1191                 jumbo = B_TRUE;
1192                 break;
1193 #endif  /* EFSYS_OPT_RX_SCATTER */
1194
1195         default:
1196                 rc = EINVAL;
1197                 goto fail4;
1198         }
1199
1200         /* Set up the new descriptor queue */
1201         EFX_POPULATE_OWORD_7(oword,
1202             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1203             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1204             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1205             FRF_AZ_RX_DESCQ_LABEL, label,
1206             FRF_AZ_RX_DESCQ_SIZE, size,
1207             FRF_AZ_RX_DESCQ_TYPE, 0,
1208             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1209
1210         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1211                             erp->er_index, &oword, B_TRUE);
1212
1213         return (0);
1214
1215 fail4:
1216         EFSYS_PROBE(fail4);
1217 fail3:
1218         EFSYS_PROBE(fail3);
1219 fail2:
1220         EFSYS_PROBE(fail2);
1221 fail1:
1222         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1223
1224         return (rc);
1225 }
1226
1227 static          void
1228 falconsiena_rx_qdestroy(
1229         __in    efx_rxq_t *erp)
1230 {
1231         efx_nic_t *enp = erp->er_enp;
1232         efx_oword_t oword;
1233
1234         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1235         --enp->en_rx_qcount;
1236
1237         /* Purge descriptor queue */
1238         EFX_ZERO_OWORD(oword);
1239
1240         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1241                             erp->er_index, &oword, B_TRUE);
1242
1243         /* Free the RXQ object */
1244         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1245 }
1246
1247 static          void
1248 falconsiena_rx_fini(
1249         __in    efx_nic_t *enp)
1250 {
1251         _NOTE(ARGUNUSED(enp))
1252 }
1253
1254 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */