sys/dev/sfxge/common/efx_rx.c

   1 /*-
   2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "efsys.h"
  35 #include "efx.h"
  36 #include "efx_types.h"
  37 #include "efx_regs.h"
  38 #include "efx_impl.h"
  39
  40
  41 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
  42
  43 static  __checkReturn   int
  44 falconsiena_rx_init(
  45         __in            efx_nic_t *enp);
  46
  47 static                  void
  48 falconsiena_rx_fini(
  49         __in            efx_nic_t *enp);
  50
  51 #if EFSYS_OPT_RX_HDR_SPLIT
  52 static  __checkReturn   int
  53 falconsiena_rx_hdr_split_enable(
  54         __in            efx_nic_t *enp,
  55         __in            unsigned int hdr_buf_size,
  56         __in            unsigned int pld_buf_size);
  57 #endif /* EFSYS_OPT_RX_HDR_SPLIT */
  58
  59 #if EFSYS_OPT_RX_SCATTER
  60 static  __checkReturn   int
  61 falconsiena_rx_scatter_enable(
  62         __in            efx_nic_t *enp,
  63         __in            unsigned int buf_size);
  64 #endif /* EFSYS_OPT_RX_SCATTER */
  65
  66 #if EFSYS_OPT_RX_SCALE
  67 static  __checkReturn   int
  68 falconsiena_rx_scale_mode_set(
  69         __in            efx_nic_t *enp,
  70         __in            efx_rx_hash_alg_t alg,
  71         __in            efx_rx_hash_type_t type,
  72         __in            boolean_t insert);
  73
  74 static  __checkReturn   int
  75 falconsiena_rx_scale_key_set(
  76         __in            efx_nic_t *enp,
  77         __in_ecount(n)  uint8_t *key,
  78         __in            size_t n);
  79
  80 static  __checkReturn   int
  81 falconsiena_rx_scale_tbl_set(
  82         __in            efx_nic_t *enp,
  83         __in_ecount(n)  unsigned int *table,
  84         __in            size_t n);
  85
  86 #endif /* EFSYS_OPT_RX_SCALE */
  87
  88 static                  void
  89 falconsiena_rx_qpost(
  90         __in            efx_rxq_t *erp,
  91         __in_ecount(n)  efsys_dma_addr_t *addrp,
  92         __in            size_t size,
  93         __in            unsigned int n,
  94         __in            unsigned int completed,
  95         __in            unsigned int added);
  96
  97 static                  void
  98 falconsiena_rx_qpush(
  99         __in            efx_rxq_t *erp,
 100         __in            unsigned int added,
 101         __inout         unsigned int *pushedp);
 102
 103 static  __checkReturn   int
 104 falconsiena_rx_qflush(
 105         __in            efx_rxq_t *erp);
 106
 107 static                  void
 108 falconsiena_rx_qenable(
 109         __in            efx_rxq_t *erp);
 110
 111 static  __checkReturn   int
 112 falconsiena_rx_qcreate(
 113         __in            efx_nic_t *enp,
 114         __in            unsigned int index,
 115         __in            unsigned int label,
 116         __in            efx_rxq_type_t type,
 117         __in            efsys_mem_t *esmp,
 118         __in            size_t n,
 119         __in            uint32_t id,
 120         __in            efx_evq_t *eep,
 121         __in            efx_rxq_t *erp);
 122
 123 static                  void
 124 falconsiena_rx_qdestroy(
 125         __in            efx_rxq_t *erp);
 126
 127 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
 128
 129
 130 #if EFSYS_OPT_FALCON
 131 static efx_rx_ops_t __efx_rx_falcon_ops = {
 132         falconsiena_rx_init,                    /* erxo_init */
 133         falconsiena_rx_fini,                    /* erxo_fini */
 134 #if EFSYS_OPT_RX_HDR_SPLIT
 135         falconsiena_rx_hdr_split_enable,        /* erxo_hdr_split_enable */
 136 #endif
 137 #if EFSYS_OPT_RX_SCATTER
 138         falconsiena_rx_scatter_enable,          /* erxo_scatter_enable */
 139 #endif
 140 #if EFSYS_OPT_RX_SCALE
 141         falconsiena_rx_scale_mode_set,          /* erxo_scale_mode_set */
 142         falconsiena_rx_scale_key_set,           /* erxo_scale_key_set */
 143         falconsiena_rx_scale_tbl_set,           /* erxo_scale_tbl_set */
 144 #endif
 145         falconsiena_rx_qpost,                   /* erxo_qpost */
 146         falconsiena_rx_qpush,                   /* erxo_qpush */
 147         falconsiena_rx_qflush,                  /* erxo_qflush */
 148         falconsiena_rx_qenable,                 /* erxo_qenable */
 149         falconsiena_rx_qcreate,                 /* erxo_qcreate */
 150         falconsiena_rx_qdestroy,                /* erxo_qdestroy */
 151 };
 152 #endif  /* EFSYS_OPT_FALCON */
 153
 154 #if EFSYS_OPT_SIENA
 155 static efx_rx_ops_t __efx_rx_siena_ops = {
 156         falconsiena_rx_init,                    /* erxo_init */
 157         falconsiena_rx_fini,                    /* erxo_fini */
 158 #if EFSYS_OPT_RX_HDR_SPLIT
 159         falconsiena_rx_hdr_split_enable,        /* erxo_hdr_split_enable */
 160 #endif
 161 #if EFSYS_OPT_RX_SCATTER
 162         falconsiena_rx_scatter_enable,          /* erxo_scatter_enable */
 163 #endif
 164 #if EFSYS_OPT_RX_SCALE
 165         falconsiena_rx_scale_mode_set,          /* erxo_scale_mode_set */
 166         falconsiena_rx_scale_key_set,           /* erxo_scale_key_set */
 167         falconsiena_rx_scale_tbl_set,           /* erxo_scale_tbl_set */
 168 #endif
 169         falconsiena_rx_qpost,                   /* erxo_qpost */
 170         falconsiena_rx_qpush,                   /* erxo_qpush */
 171         falconsiena_rx_qflush,                  /* erxo_qflush */
 172         falconsiena_rx_qenable,                 /* erxo_qenable */
 173         falconsiena_rx_qcreate,                 /* erxo_qcreate */
 174         falconsiena_rx_qdestroy,                /* erxo_qdestroy */
 175 };
 176 #endif  /* EFSYS_OPT_SIENA */
 177
 178 #if EFSYS_OPT_HUNTINGTON
 179 static efx_rx_ops_t __efx_rx_hunt_ops = {
 180         hunt_rx_init,                           /* erxo_init */
 181         hunt_rx_fini,                           /* erxo_fini */
 182 #if EFSYS_OPT_RX_HDR_SPLIT
 183         hunt_rx_hdr_split_enable,               /* erxo_hdr_split_enable */
 184 #endif
 185 #if EFSYS_OPT_RX_SCATTER
 186         hunt_rx_scatter_enable,                 /* erxo_scatter_enable */
 187 #endif
 188 #if EFSYS_OPT_RX_SCALE
 189         hunt_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 190         hunt_rx_scale_key_set,                  /* erxo_scale_key_set */
 191         hunt_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 192 #endif
 193         hunt_rx_qpost,                          /* erxo_qpost */
 194         hunt_rx_qpush,                          /* erxo_qpush */
 195         hunt_rx_qflush,                         /* erxo_qflush */
 196         hunt_rx_qenable,                        /* erxo_qenable */
 197         hunt_rx_qcreate,                        /* erxo_qcreate */
 198         hunt_rx_qdestroy,                       /* erxo_qdestroy */
 199 };
 200 #endif  /* EFSYS_OPT_HUNTINGTON */
 201
 202
 203         __checkReturn   int
 204 efx_rx_init(
 205         __inout         efx_nic_t *enp)
 206 {
 207         efx_rx_ops_t *erxop;
 208         int rc;
 209
 210         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 211         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 212
 213         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 214                 rc = EINVAL;
 215                 goto fail1;
 216         }
 217
 218         if (enp->en_mod_flags & EFX_MOD_RX) {
 219                 rc = EINVAL;
 220                 goto fail2;
 221         }
 222
 223         switch (enp->en_family) {
 224 #if EFSYS_OPT_FALCON
 225         case EFX_FAMILY_FALCON:
 226                 erxop = (efx_rx_ops_t *)&__efx_rx_falcon_ops;
 227                 break;
 228 #endif /* EFSYS_OPT_FALCON */
 229
 230 #if EFSYS_OPT_SIENA
 231         case EFX_FAMILY_SIENA:
 232                 erxop = (efx_rx_ops_t *)&__efx_rx_siena_ops;
 233                 break;
 234 #endif /* EFSYS_OPT_SIENA */
 235
 236 #if EFSYS_OPT_HUNTINGTON
 237         case EFX_FAMILY_HUNTINGTON:
 238                 erxop = (efx_rx_ops_t *)&__efx_rx_hunt_ops;
 239                 break;
 240 #endif /* EFSYS_OPT_HUNTINGTON */
 241
 242         default:
 243                 EFSYS_ASSERT(0);
 244                 rc = ENOTSUP;
 245                 goto fail3;
 246         }
 247
 248         if ((rc = erxop->erxo_init(enp)) != 0)
 249                 goto fail4;
 250
 251         enp->en_erxop = erxop;
 252         enp->en_mod_flags |= EFX_MOD_RX;
 253         return (0);
 254
 255 fail4:
 256         EFSYS_PROBE(fail4);
 257 fail3:
 258         EFSYS_PROBE(fail3);
 259 fail2:
 260         EFSYS_PROBE(fail2);
 261 fail1:
 262         EFSYS_PROBE1(fail1, int, rc);
 263
 264         enp->en_erxop = NULL;
 265         enp->en_mod_flags &= ~EFX_MOD_RX;
 266         return (rc);
 267 }
 268
 269                         void
 270 efx_rx_fini(
 271         __in            efx_nic_t *enp)
 272 {
 273         efx_rx_ops_t *erxop = enp->en_erxop;
 274
 275         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 276         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 277         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 278         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 279
 280         erxop->erxo_fini(enp);
 281
 282         enp->en_erxop = NULL;
 283         enp->en_mod_flags &= ~EFX_MOD_RX;
 284 }
 285
 286 #if EFSYS_OPT_RX_HDR_SPLIT
 287         __checkReturn   int
 288 efx_rx_hdr_split_enable(
 289         __in            efx_nic_t *enp,
 290         __in            unsigned int hdr_buf_size,
 291         __in            unsigned int pld_buf_size)
 292 {
 293         efx_rx_ops_t *erxop = enp->en_erxop;
 294         int rc;
 295
 296         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 297         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 298         EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
 299
 300         if ((rc = erxop->erxo_hdr_split_enable(enp, hdr_buf_size,
 301             pld_buf_size)) != 0)
 302                 goto fail1;
 303
 304         return (0);
 305
 306 fail1:
 307         EFSYS_PROBE1(fail1, int, rc);
 308         return (rc);
 309 }
 310 #endif  /* EFSYS_OPT_RX_HDR_SPLIT */
 311
 312 #if EFSYS_OPT_RX_SCATTER
 313         __checkReturn   int
 314 efx_rx_scatter_enable(
 315         __in            efx_nic_t *enp,
 316         __in            unsigned int buf_size)
 317 {
 318         efx_rx_ops_t *erxop = enp->en_erxop;
 319         int rc;
 320
 321         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 322         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 323
 324         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 325                 goto fail1;
 326
 327         return (0);
 328
 329 fail1:
 330         EFSYS_PROBE1(fail1, int, rc);
 331         return (rc);
 332 }
 333 #endif  /* EFSYS_OPT_RX_SCATTER */
 334
 335 #if EFSYS_OPT_RX_SCALE
 336         __checkReturn   int
 337 efx_rx_hash_support_get(
 338         __in            efx_nic_t *enp,
 339         __out           efx_rx_hash_support_t *supportp)
 340 {
 341         int rc;
 342
 343         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 344         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 345
 346         if (supportp == NULL) {
 347                 rc = EINVAL;
 348                 goto fail1;
 349         }
 350
 351         /* Report if resources are available to insert RX hash value */
 352         *supportp = enp->en_hash_support;
 353
 354         return (0);
 355
 356 fail1:
 357         EFSYS_PROBE1(fail1, int, rc);
 358
 359         return (rc);
 360 }
 361
 362         __checkReturn   int
 363 efx_rx_scale_support_get(
 364         __in            efx_nic_t *enp,
 365         __out           efx_rx_scale_support_t *supportp)
 366 {
 367         int rc;
 368
 369         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 370         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 371
 372         if (supportp == NULL) {
 373                 rc = EINVAL;
 374                 goto fail1;
 375         }
 376
 377         /* Report if resources are available to support RSS */
 378         *supportp = enp->en_rss_support;
 379
 380         return (0);
 381
 382 fail1:
 383         EFSYS_PROBE1(fail1, int, rc);
 384
 385         return (rc);
 386 }
 387
 388         __checkReturn   int
 389 efx_rx_scale_mode_set(
 390         __in            efx_nic_t *enp,
 391         __in            efx_rx_hash_alg_t alg,
 392         __in            efx_rx_hash_type_t type,
 393         __in            boolean_t insert)
 394 {
 395         efx_rx_ops_t *erxop = enp->en_erxop;
 396         int rc;
 397
 398         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 399         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 400
 401         if (erxop->erxo_scale_mode_set != NULL) {
 402                 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
 403                             type, insert)) != 0)
 404                         goto fail1;
 405         }
 406
 407         return (0);
 408
 409 fail1:
 410         EFSYS_PROBE1(fail1, int, rc);
 411         return (rc);
 412 }
 413 #endif  /* EFSYS_OPT_RX_SCALE */
 414
 415 #if EFSYS_OPT_RX_SCALE
 416         __checkReturn   int
 417 efx_rx_scale_key_set(
 418         __in            efx_nic_t *enp,
 419         __in_ecount(n)  uint8_t *key,
 420         __in            size_t n)
 421 {
 422         efx_rx_ops_t *erxop = enp->en_erxop;
 423         int rc;
 424
 425         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 426         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 427
 428         if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
 429                 goto fail1;
 430
 431         return (0);
 432
 433 fail1:
 434         EFSYS_PROBE1(fail1, int, rc);
 435
 436         return (rc);
 437 }
 438 #endif  /* EFSYS_OPT_RX_SCALE */
 439
 440 #if EFSYS_OPT_RX_SCALE
 441         __checkReturn   int
 442 efx_rx_scale_tbl_set(
 443         __in            efx_nic_t *enp,
 444         __in_ecount(n)  unsigned int *table,
 445         __in            size_t n)
 446 {
 447         efx_rx_ops_t *erxop = enp->en_erxop;
 448         int rc;
 449
 450         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 451         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 452
 453         if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
 454                 goto fail1;
 455
 456         return (0);
 457
 458 fail1:
 459         EFSYS_PROBE1(fail1, int, rc);
 460
 461         return (rc);
 462 }
 463 #endif  /* EFSYS_OPT_RX_SCALE */
 464
 465                         void
 466 efx_rx_qpost(
 467         __in            efx_rxq_t *erp,
 468         __in_ecount(n)  efsys_dma_addr_t *addrp,
 469         __in            size_t size,
 470         __in            unsigned int n,
 471         __in            unsigned int completed,
 472         __in            unsigned int added)
 473 {
 474         efx_nic_t *enp = erp->er_enp;
 475         efx_rx_ops_t *erxop = enp->en_erxop;
 476
 477         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 478
 479         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 480 }
 481
 482                         void
 483 efx_rx_qpush(
 484         __in            efx_rxq_t *erp,
 485         __in            unsigned int added,
 486         __inout         unsigned int *pushedp)
 487 {
 488         efx_nic_t *enp = erp->er_enp;
 489         efx_rx_ops_t *erxop = enp->en_erxop;
 490
 491         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 492
 493         erxop->erxo_qpush(erp, added, pushedp);
 494 }
 495
 496         __checkReturn   int
 497 efx_rx_qflush(
 498         __in            efx_rxq_t *erp)
 499 {
 500         efx_nic_t *enp = erp->er_enp;
 501         efx_rx_ops_t *erxop = enp->en_erxop;
 502         int rc;
 503
 504         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 505
 506         if ((rc = erxop->erxo_qflush(erp)) != 0)
 507                 goto fail1;
 508
 509         return (0);
 510
 511 fail1:
 512         EFSYS_PROBE1(fail1, int, rc);
 513
 514         return (rc);
 515 }
 516
 517                         void
 518 efx_rx_qenable(
 519         __in            efx_rxq_t *erp)
 520 {
 521         efx_nic_t *enp = erp->er_enp;
 522         efx_rx_ops_t *erxop = enp->en_erxop;
 523
 524         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 525
 526         erxop->erxo_qenable(erp);
 527 }
 528
 529         __checkReturn   int
 530 efx_rx_qcreate(
 531         __in            efx_nic_t *enp,
 532         __in            unsigned int index,
 533         __in            unsigned int label,
 534         __in            efx_rxq_type_t type,
 535         __in            efsys_mem_t *esmp,
 536         __in            size_t n,
 537         __in            uint32_t id,
 538         __in            efx_evq_t *eep,
 539         __deref_out     efx_rxq_t **erpp)
 540 {
 541         efx_rx_ops_t *erxop = enp->en_erxop;
 542         efx_rxq_t *erp;
 543         int rc;
 544
 545         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 546         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 547
 548         /* Allocate an RXQ object */
 549         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 550
 551         if (erp == NULL) {
 552                 rc = ENOMEM;
 553                 goto fail1;
 554         }
 555
 556         erp->er_magic = EFX_RXQ_MAGIC;
 557         erp->er_enp = enp;
 558         erp->er_index = index;
 559         erp->er_mask = n - 1;
 560         erp->er_esmp = esmp;
 561
 562         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 563             eep, erp)) != 0)
 564                 goto fail2;
 565
 566         enp->en_rx_qcount++;
 567         *erpp = erp;
 568
 569         return (0);
 570
 571 fail2:
 572         EFSYS_PROBE(fail2);
 573
 574         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 575 fail1:
 576         EFSYS_PROBE1(fail1, int, rc);
 577
 578         return (rc);
 579 }
 580
 581                         void
 582 efx_rx_qdestroy(
 583         __in            efx_rxq_t *erp)
 584 {
 585         efx_nic_t *enp = erp->er_enp;
 586         efx_rx_ops_t *erxop = enp->en_erxop;
 587
 588         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 589
 590         erxop->erxo_qdestroy(erp);
 591 }
 592
 593 /*
 594  * Psuedo-header info for Siena/Falcon.
 595  *
 596  * The psuedo-header is a byte array of one of the forms:
 597  *
 598  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
 599  * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.TT.TT.TT.TT
 600  * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.LL.LL
 601  *
 602  * where:
 603  *
 604  * TT.TT.TT.TT is a 32-bit Toeplitz hash
 605  * LL.LL is a 16-bit LFSR hash
 606  *
 607  * Hash values are in network (big-endian) byte order.
 608  *
 609  *
 610  * On Huntington the pseudo-header is laid out as:
 611  * (See also SF-109306-TC section 9)
 612  *
 613  * Toeplitz hash (32 bits, little-endian)
 614  * Out-of-band outer VLAN tag
 615  *     (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag)
 616  * Out-of-band inner VLAN tag
 617  *     (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag)
 618  * Packet length (16 bits, little-endian, may be 0)
 619  * MAC timestamp (32 bits, little-endian, may be 0)
 620  * VLAN tag
 621  *     (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag)
 622  * VLAN tag
 623  *     (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag)
 624  */
 625
 626         __checkReturn   int
 627 efx_psuedo_hdr_pkt_length_get(
 628         __in            efx_nic_t *enp,
 629         __in            uint8_t *buffer,
 630         __out           uint16_t *pkt_lengthp)
 631 {
 632         if (enp->en_family != EFX_FAMILY_HUNTINGTON) {
 633                 EFSYS_ASSERT(0);
 634                 return (ENOTSUP);
 635         }
 636
 637         *pkt_lengthp = buffer[8] | (buffer[9] << 8);
 638
 639         return (0);
 640 }
 641
 642 #if EFSYS_OPT_RX_SCALE
 643
 644 uint32_t
 645 efx_psuedo_hdr_hash_get(
 646         __in            efx_nic_t *enp,
 647         __in            efx_rx_hash_alg_t func,
 648         __in            uint8_t *buffer)
 649 {
 650         if (func == EFX_RX_HASHALG_TOEPLITZ) {
 651                 switch (enp->en_family) {
 652                 case EFX_FAMILY_FALCON:
 653                 case EFX_FAMILY_SIENA:
 654                         return ((buffer[12] << 24) |
 655                             (buffer[13] << 16) |
 656                             (buffer[14] << 8) |
 657                             buffer[15]);
 658                 case EFX_FAMILY_HUNTINGTON:
 659                         return (buffer[0] |
 660                             (buffer[1] << 8) |
 661                             (buffer[2] << 16) |
 662                             (buffer[3] << 24));
 663                 default:
 664                         EFSYS_ASSERT(0);
 665                         return (0);
 666                 }
 667         } else if (func == EFX_RX_HASHALG_LFSR) {
 668                 EFSYS_ASSERT(enp->en_family == EFX_FAMILY_FALCON ||
 669                     enp->en_family == EFX_FAMILY_SIENA);
 670                 return ((buffer[14] << 8) | buffer[15]);
 671         } else {
 672                 EFSYS_ASSERT(0);
 673                 return (0);
 674         }
 675 }
 676
 677 #endif  /* EFSYS_OPT_RX_SCALE */
 678
 679 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
 680
 681 static  __checkReturn   int
 682 falconsiena_rx_init(
 683         __in            efx_nic_t *enp)
 684 {
 685         efx_oword_t oword;
 686         unsigned int index;
 687
 688         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 689
 690         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 691         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 692         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 693         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 694         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 695         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 696         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 697
 698         /* Zero the RSS table */
 699         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 700             index++) {
 701                 EFX_ZERO_OWORD(oword);
 702                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 703                                     index, &oword, B_TRUE);
 704         }
 705
 706 #if EFSYS_OPT_RX_SCALE
 707         /* The RSS key and indirection table are writable. */
 708         enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
 709
 710         /* Hardware can insert RX hash with/without RSS */
 711         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 712 #endif  /* EFSYS_OPT_RX_SCALE */
 713
 714         return (0);
 715 }
 716
 717 #if EFSYS_OPT_RX_HDR_SPLIT
 718 static  __checkReturn   int
 719 falconsiena_rx_hdr_split_enable(
 720         __in            efx_nic_t *enp,
 721         __in            unsigned int hdr_buf_size,
 722         __in            unsigned int pld_buf_size)
 723 {
 724         unsigned int nhdr32;
 725         unsigned int npld32;
 726         efx_oword_t oword;
 727         int rc;
 728
 729         nhdr32 = hdr_buf_size / 32;
 730         if ((nhdr32 == 0) ||
 731             (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
 732             ((hdr_buf_size % 32) != 0)) {
 733                 rc = EINVAL;
 734                 goto fail1;
 735         }
 736
 737         npld32 = pld_buf_size / 32;
 738         if ((npld32 == 0) ||
 739             (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
 740             ((pld_buf_size % 32) != 0)) {
 741                 rc = EINVAL;
 742                 goto fail2;
 743         }
 744
 745         if (enp->en_rx_qcount > 0) {
 746                 rc = EBUSY;
 747                 goto fail3;
 748         }
 749
 750         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 751
 752         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
 753         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
 754         EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
 755
 756         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 757
 758         return (0);
 759
 760 fail3:
 761         EFSYS_PROBE(fail3);
 762 fail2:
 763         EFSYS_PROBE(fail2);
 764 fail1:
 765         EFSYS_PROBE1(fail1, int, rc);
 766
 767         return (rc);
 768 }
 769 #endif  /* EFSYS_OPT_RX_HDR_SPLIT */
 770
 771 #if EFSYS_OPT_RX_SCATTER
 772 static  __checkReturn   int
 773 falconsiena_rx_scatter_enable(
 774         __in            efx_nic_t *enp,
 775         __in            unsigned int buf_size)
 776 {
 777         unsigned int nbuf32;
 778         efx_oword_t oword;
 779         int rc;
 780
 781         nbuf32 = buf_size / 32;
 782         if ((nbuf32 == 0) ||
 783             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 784             ((buf_size % 32) != 0)) {
 785                 rc = EINVAL;
 786                 goto fail1;
 787         }
 788
 789         if (enp->en_rx_qcount > 0) {
 790                 rc = EBUSY;
 791                 goto fail2;
 792         }
 793
 794         /* Set scatter buffer size */
 795         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 796         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 797         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 798
 799         /* Enable scatter for packets not matching a filter */
 800         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 801         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 802         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 803
 804         return (0);
 805
 806 fail2:
 807         EFSYS_PROBE(fail2);
 808 fail1:
 809         EFSYS_PROBE1(fail1, int, rc);
 810
 811         return (rc);
 812 }
 813 #endif  /* EFSYS_OPT_RX_SCATTER */
 814
 815
 816 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 817         do {                                                            \
 818                 efx_oword_t oword;                                      \
 819                                                                         \
 820                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 821                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 822                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 823                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 824                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 825                     (_insert) ? 1 : 0);                                 \
 826                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 827                                                                         \
 828                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 829                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 830                             &oword);                                    \
 831                         EFX_SET_OWORD_FIELD(oword,                      \
 832                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 833                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 834                             &oword);                                    \
 835                 }                                                       \
 836                                                                         \
 837                 _NOTE(CONSTANTCONDITION)                                \
 838         } while (B_FALSE)
 839
 840 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 841         do {                                                            \
 842                 efx_oword_t oword;                                      \
 843                                                                         \
 844                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 845                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 846                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 847                     (_ip) ? 1 : 0);                                     \
 848                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 849                     (_tcp) ? 0 : 1);                                    \
 850                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 851                     (_insert) ? 1 : 0);                                 \
 852                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 853                                                                         \
 854                 _NOTE(CONSTANTCONDITION)                                \
 855         } while (B_FALSE)
 856
 857 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 858         do {                                                            \
 859                 efx_oword_t oword;                                      \
 860                                                                         \
 861                 if ((_enp)->en_family == EFX_FAMILY_FALCON) {           \
 862                         (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0;        \
 863                         break;                                          \
 864                 }                                                       \
 865                                                                         \
 866                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 867                 EFX_SET_OWORD_FIELD(oword,                              \
 868                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 869                 EFX_SET_OWORD_FIELD(oword,                              \
 870                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 871                 EFX_SET_OWORD_FIELD(oword,                              \
 872                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 873                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 874                                                                         \
 875                 (_rc) = 0;                                              \
 876                                                                         \
 877                 _NOTE(CONSTANTCONDITION)                                \
 878         } while (B_FALSE)
 879
 880
 881 #if EFSYS_OPT_RX_SCALE
 882
 883 static  __checkReturn   int
 884 falconsiena_rx_scale_mode_set(
 885         __in            efx_nic_t *enp,
 886         __in            efx_rx_hash_alg_t alg,
 887         __in            efx_rx_hash_type_t type,
 888         __in            boolean_t insert)
 889 {
 890         int rc;
 891
 892         switch (alg) {
 893         case EFX_RX_HASHALG_LFSR:
 894                 EFX_RX_LFSR_HASH(enp, insert);
 895                 break;
 896
 897         case EFX_RX_HASHALG_TOEPLITZ:
 898                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 899                     type & (1 << EFX_RX_HASH_IPV4),
 900                     type & (1 << EFX_RX_HASH_TCPIPV4));
 901
 902                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 903                     type & (1 << EFX_RX_HASH_IPV6),
 904                     type & (1 << EFX_RX_HASH_TCPIPV6),
 905                     rc);
 906                 if (rc != 0)
 907                         goto fail1;
 908
 909                 break;
 910
 911         default:
 912                 rc = EINVAL;
 913                 goto fail2;
 914         }
 915
 916         return (0);
 917
 918 fail2:
 919         EFSYS_PROBE(fail2);
 920 fail1:
 921         EFSYS_PROBE1(fail1, int, rc);
 922
 923         EFX_RX_LFSR_HASH(enp, B_FALSE);
 924
 925         return (rc);
 926 }
 927 #endif
 928
 929 #if EFSYS_OPT_RX_SCALE
 930 static  __checkReturn   int
 931 falconsiena_rx_scale_key_set(
 932         __in            efx_nic_t *enp,
 933         __in_ecount(n)  uint8_t *key,
 934         __in            size_t n)
 935 {
 936         efx_oword_t oword;
 937         unsigned int byte;
 938         unsigned int offset;
 939         int rc;
 940
 941         byte = 0;
 942
 943         /* Write Toeplitz IPv4 hash key */
 944         EFX_ZERO_OWORD(oword);
 945         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 946             offset > 0 && byte < n;
 947             --offset)
 948                 oword.eo_u8[offset - 1] = key[byte++];
 949
 950         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 951
 952         byte = 0;
 953
 954         /* Verify Toeplitz IPv4 hash key */
 955         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 956         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 957             offset > 0 && byte < n;
 958             --offset) {
 959                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 960                         rc = EFAULT;
 961                         goto fail1;
 962                 }
 963         }
 964
 965         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 966                 goto done;
 967
 968         EFSYS_ASSERT3U(enp->en_family, !=, EFX_FAMILY_FALCON);
 969
 970         byte = 0;
 971
 972         /* Write Toeplitz IPv6 hash key 3 */
 973         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 974         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 975             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 976             offset > 0 && byte < n;
 977             --offset)
 978                 oword.eo_u8[offset - 1] = key[byte++];
 979
 980         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 981
 982         /* Write Toeplitz IPv6 hash key 2 */
 983         EFX_ZERO_OWORD(oword);
 984         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 985             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 986             offset > 0 && byte < n;
 987             --offset)
 988                 oword.eo_u8[offset - 1] = key[byte++];
 989
 990         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 991
 992         /* Write Toeplitz IPv6 hash key 1 */
 993         EFX_ZERO_OWORD(oword);
 994         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 995             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 996             offset > 0 && byte < n;
 997             --offset)
 998                 oword.eo_u8[offset - 1] = key[byte++];
 999
1000         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1001
1002         byte = 0;
1003
1004         /* Verify Toeplitz IPv6 hash key 3 */
1005         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1006         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1007             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1008             offset > 0 && byte < n;
1009             --offset) {
1010                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1011                         rc = EFAULT;
1012                         goto fail2;
1013                 }
1014         }
1015
1016         /* Verify Toeplitz IPv6 hash key 2 */
1017         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1018         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1019             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1020             offset > 0 && byte < n;
1021             --offset) {
1022                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1023                         rc = EFAULT;
1024                         goto fail3;
1025                 }
1026         }
1027
1028         /* Verify Toeplitz IPv6 hash key 1 */
1029         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1030         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1031             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1032             offset > 0 && byte < n;
1033             --offset) {
1034                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1035                         rc = EFAULT;
1036                         goto fail4;
1037                 }
1038         }
1039
1040 done:
1041         return (0);
1042
1043 fail4:
1044         EFSYS_PROBE(fail4);
1045 fail3:
1046         EFSYS_PROBE(fail3);
1047 fail2:
1048         EFSYS_PROBE(fail2);
1049 fail1:
1050         EFSYS_PROBE1(fail1, int, rc);
1051
1052         return (rc);
1053 }
1054 #endif
1055
1056 #if EFSYS_OPT_RX_SCALE
1057 static  __checkReturn   int
1058 falconsiena_rx_scale_tbl_set(
1059         __in            efx_nic_t *enp,
1060         __in_ecount(n)  unsigned int *table,
1061         __in            size_t n)
1062 {
1063         efx_oword_t oword;
1064         int index;
1065         int rc;
1066
1067         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1068         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1069
1070         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1071                 rc = EINVAL;
1072                 goto fail1;
1073         }
1074
1075         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1076                 uint32_t byte;
1077
1078                 /* Calculate the entry to place in the table */
1079                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1080
1081                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1082
1083                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1084
1085                 /* Write the table */
1086                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1087                                     index, &oword, B_TRUE);
1088         }
1089
1090         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1091                 uint32_t byte;
1092
1093                 /* Determine if we're starting a new batch */
1094                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1095
1096                 /* Read the table */
1097                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1098                                     index, &oword, B_TRUE);
1099
1100                 /* Verify the entry */
1101                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1102                         rc = EFAULT;
1103                         goto fail2;
1104                 }
1105         }
1106
1107         return (0);
1108
1109 fail2:
1110         EFSYS_PROBE(fail2);
1111 fail1:
1112         EFSYS_PROBE1(fail1, int, rc);
1113
1114         return (rc);
1115 }
1116 #endif
1117
1118 static                  void
1119 falconsiena_rx_qpost(
1120         __in            efx_rxq_t *erp,
1121         __in_ecount(n)  efsys_dma_addr_t *addrp,
1122         __in            size_t size,
1123         __in            unsigned int n,
1124         __in            unsigned int completed,
1125         __in            unsigned int added)
1126 {
1127         efx_qword_t qword;
1128         unsigned int i;
1129         unsigned int offset;
1130         unsigned int id;
1131
1132         /* The client driver must not overfill the queue */
1133         EFSYS_ASSERT3U(added - completed + n, <=,
1134             EFX_RXQ_LIMIT(erp->er_mask + 1));
1135
1136         id = added & (erp->er_mask);
1137         for (i = 0; i < n; i++) {
1138                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1139                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1140                     size_t, size);
1141
1142                 EFX_POPULATE_QWORD_3(qword,
1143                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1144                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1145                     (uint32_t)(addrp[i] & 0xffffffff),
1146                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1147                     (uint32_t)(addrp[i] >> 32));
1148
1149                 offset = id * sizeof (efx_qword_t);
1150                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1151
1152                 id = (id + 1) & (erp->er_mask);
1153         }
1154 }
1155
1156 static                  void
1157 falconsiena_rx_qpush(
1158         __in    efx_rxq_t *erp,
1159         __in    unsigned int added,
1160         __inout unsigned int *pushedp)
1161 {
1162         efx_nic_t *enp = erp->er_enp;
1163         unsigned int pushed = *pushedp;
1164         uint32_t wptr;
1165         efx_oword_t oword;
1166         efx_dword_t dword;
1167
1168         /* All descriptors are pushed */
1169         *pushedp = added;
1170
1171         /* Push the populated descriptors out */
1172         wptr = added & erp->er_mask;
1173
1174         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1175
1176         /* Only write the third DWORD */
1177         EFX_POPULATE_DWORD_1(dword,
1178             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1179
1180         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1181         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1182             wptr, pushed & erp->er_mask);
1183         EFSYS_PIO_WRITE_BARRIER();
1184         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1185                             erp->er_index, &dword, B_FALSE);
1186 }
1187
1188 static  __checkReturn   int
1189 falconsiena_rx_qflush(
1190         __in    efx_rxq_t *erp)
1191 {
1192         efx_nic_t *enp = erp->er_enp;
1193         efx_oword_t oword;
1194         uint32_t label;
1195
1196         label = erp->er_index;
1197
1198         /* Flush the queue */
1199         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1200             FRF_AZ_RX_FLUSH_DESCQ, label);
1201         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1202
1203         return (0);
1204 }
1205
1206 static          void
1207 falconsiena_rx_qenable(
1208         __in    efx_rxq_t *erp)
1209 {
1210         efx_nic_t *enp = erp->er_enp;
1211         efx_oword_t oword;
1212
1213         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1214
1215         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1216                             erp->er_index, &oword, B_TRUE);
1217
1218         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1219         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1220         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1221
1222         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1223                             erp->er_index, &oword, B_TRUE);
1224 }
1225
1226 static  __checkReturn   int
1227 falconsiena_rx_qcreate(
1228         __in            efx_nic_t *enp,
1229         __in            unsigned int index,
1230         __in            unsigned int label,
1231         __in            efx_rxq_type_t type,
1232         __in            efsys_mem_t *esmp,
1233         __in            size_t n,
1234         __in            uint32_t id,
1235         __in            efx_evq_t *eep,
1236         __in            efx_rxq_t *erp)
1237 {
1238         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1239         efx_oword_t oword;
1240         uint32_t size;
1241         boolean_t split;
1242         boolean_t jumbo;
1243         int rc;
1244
1245         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1246             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1247         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1248         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1249
1250         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1251         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1252
1253         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1254                 rc = EINVAL;
1255                 goto fail1;
1256         }
1257         if (index >= encp->enc_rxq_limit) {
1258                 rc = EINVAL;
1259                 goto fail2;
1260         }
1261         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1262             size++)
1263                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1264                         break;
1265         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1266                 rc = EINVAL;
1267                 goto fail3;
1268         }
1269
1270         switch (type) {
1271         case EFX_RXQ_TYPE_DEFAULT:
1272                 split = B_FALSE;
1273                 jumbo = B_FALSE;
1274                 break;
1275
1276 #if EFSYS_OPT_RX_HDR_SPLIT
1277         case EFX_RXQ_TYPE_SPLIT_HEADER:
1278                 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
1279                         rc = EINVAL;
1280                         goto fail4;
1281                 }
1282                 split = B_TRUE;
1283                 jumbo = B_TRUE;
1284                 break;
1285
1286         case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
1287                 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
1288                         rc = EINVAL;
1289                         goto fail4;
1290                 }
1291                 split = B_FALSE;
1292                 jumbo = B_TRUE;
1293                 break;
1294 #endif  /* EFSYS_OPT_RX_HDR_SPLIT */
1295
1296 #if EFSYS_OPT_RX_SCATTER
1297         case EFX_RXQ_TYPE_SCATTER:
1298                 if (enp->en_family < EFX_FAMILY_SIENA) {
1299                         rc = EINVAL;
1300                         goto fail4;
1301                 }
1302                 split = B_FALSE;
1303                 jumbo = B_TRUE;
1304                 break;
1305 #endif  /* EFSYS_OPT_RX_SCATTER */
1306
1307         default:
1308                 rc = EINVAL;
1309                 goto fail4;
1310         }
1311
1312         /* Set up the new descriptor queue */
1313         EFX_POPULATE_OWORD_10(oword,
1314             FRF_CZ_RX_HDR_SPLIT, split,
1315             FRF_AZ_RX_ISCSI_DDIG_EN, 0,
1316             FRF_AZ_RX_ISCSI_HDIG_EN, 0,
1317             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1318             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1319             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1320             FRF_AZ_RX_DESCQ_LABEL, label,
1321             FRF_AZ_RX_DESCQ_SIZE, size,
1322             FRF_AZ_RX_DESCQ_TYPE, 0,
1323             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1324
1325         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1326                             erp->er_index, &oword, B_TRUE);
1327
1328         return (0);
1329
1330 fail4:
1331         EFSYS_PROBE(fail4);
1332 fail3:
1333         EFSYS_PROBE(fail3);
1334 fail2:
1335         EFSYS_PROBE(fail2);
1336 fail1:
1337         EFSYS_PROBE1(fail1, int, rc);
1338
1339         return (rc);
1340 }
1341
1342 static          void
1343 falconsiena_rx_qdestroy(
1344         __in    efx_rxq_t *erp)
1345 {
1346         efx_nic_t *enp = erp->er_enp;
1347         efx_oword_t oword;
1348
1349         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1350         --enp->en_rx_qcount;
1351
1352         /* Purge descriptor queue */
1353         EFX_ZERO_OWORD(oword);
1354
1355         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1356                             erp->er_index, &oword, B_TRUE);
1357
1358         /* Free the RXQ object */
1359         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1360 }
1361
1362 static          void
1363 falconsiena_rx_fini(
1364         __in    efx_nic_t *enp)
1365 {
1366         _NOTE(ARGUNUSED(enp))
1367 }
1368
1369 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */