sys/dev/sfxge/common/efx_ev.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include "efx.h"
  37 #include "efx_impl.h"
  38 #if EFSYS_OPT_MON_MCDI
  39 #include "mcdi_mon.h"
  40 #endif
  41
  42 #if EFSYS_OPT_QSTATS
  43 #define EFX_EV_QSTAT_INCR(_eep, _stat)                                  \
  44         do {                                                            \
  45                 (_eep)->ee_stat[_stat]++;                               \
  46         _NOTE(CONSTANTCONDITION)                                        \
  47         } while (B_FALSE)
  48 #else
  49 #define EFX_EV_QSTAT_INCR(_eep, _stat)
  50 #endif
  51
  52 #define EFX_EV_PRESENT(_qword)                                          \
  53         (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&        \
  54         EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
  55
  56
  57
  58 #if EFSYS_OPT_SIENA
  59
  60 static  __checkReturn   efx_rc_t
  61 siena_ev_init(
  62         __in            efx_nic_t *enp);
  63
  64 static                  void
  65 siena_ev_fini(
  66         __in            efx_nic_t *enp);
  67
  68 static  __checkReturn   efx_rc_t
  69 siena_ev_qcreate(
  70         __in            efx_nic_t *enp,
  71         __in            unsigned int index,
  72         __in            efsys_mem_t *esmp,
  73         __in            size_t n,
  74         __in            uint32_t id,
  75         __in            uint32_t us,
  76         __in            uint32_t flags,
  77         __in            efx_evq_t *eep);
  78
  79 static                  void
  80 siena_ev_qdestroy(
  81         __in            efx_evq_t *eep);
  82
  83 static  __checkReturn   efx_rc_t
  84 siena_ev_qprime(
  85         __in            efx_evq_t *eep,
  86         __in            unsigned int count);
  87
  88 static                  void
  89 siena_ev_qpost(
  90         __in    efx_evq_t *eep,
  91         __in    uint16_t data);
  92
  93 static  __checkReturn   efx_rc_t
  94 siena_ev_qmoderate(
  95         __in            efx_evq_t *eep,
  96         __in            unsigned int us);
  97
  98 #if EFSYS_OPT_QSTATS
  99 static                  void
 100 siena_ev_qstats_update(
 101         __in                            efx_evq_t *eep,
 102         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat);
 103
 104 #endif
 105
 106 #endif /* EFSYS_OPT_SIENA */
 107
 108 #if EFSYS_OPT_SIENA
 109 static const efx_ev_ops_t       __efx_ev_siena_ops = {
 110         siena_ev_init,                          /* eevo_init */
 111         siena_ev_fini,                          /* eevo_fini */
 112         siena_ev_qcreate,                       /* eevo_qcreate */
 113         siena_ev_qdestroy,                      /* eevo_qdestroy */
 114         siena_ev_qprime,                        /* eevo_qprime */
 115         siena_ev_qpost,                         /* eevo_qpost */
 116         siena_ev_qmoderate,                     /* eevo_qmoderate */
 117 #if EFSYS_OPT_QSTATS
 118         siena_ev_qstats_update,                 /* eevo_qstats_update */
 119 #endif
 120 };
 121 #endif /* EFSYS_OPT_SIENA */
 122
 123 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 124 static const efx_ev_ops_t       __efx_ev_ef10_ops = {
 125         ef10_ev_init,                           /* eevo_init */
 126         ef10_ev_fini,                           /* eevo_fini */
 127         ef10_ev_qcreate,                        /* eevo_qcreate */
 128         ef10_ev_qdestroy,                       /* eevo_qdestroy */
 129         ef10_ev_qprime,                         /* eevo_qprime */
 130         ef10_ev_qpost,                          /* eevo_qpost */
 131         ef10_ev_qmoderate,                      /* eevo_qmoderate */
 132 #if EFSYS_OPT_QSTATS
 133         ef10_ev_qstats_update,                  /* eevo_qstats_update */
 134 #endif
 135 };
 136 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 137
 138
 139         __checkReturn   efx_rc_t
 140 efx_ev_init(
 141         __in            efx_nic_t *enp)
 142 {
 143         const efx_ev_ops_t *eevop;
 144         efx_rc_t rc;
 145
 146         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 147         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 148
 149         if (enp->en_mod_flags & EFX_MOD_EV) {
 150                 rc = EINVAL;
 151                 goto fail1;
 152         }
 153
 154         switch (enp->en_family) {
 155 #if EFSYS_OPT_SIENA
 156         case EFX_FAMILY_SIENA:
 157                 eevop = &__efx_ev_siena_ops;
 158                 break;
 159 #endif /* EFSYS_OPT_SIENA */
 160
 161 #if EFSYS_OPT_HUNTINGTON
 162         case EFX_FAMILY_HUNTINGTON:
 163                 eevop = &__efx_ev_ef10_ops;
 164                 break;
 165 #endif /* EFSYS_OPT_HUNTINGTON */
 166
 167 #if EFSYS_OPT_MEDFORD
 168         case EFX_FAMILY_MEDFORD:
 169                 eevop = &__efx_ev_ef10_ops;
 170                 break;
 171 #endif /* EFSYS_OPT_MEDFORD */
 172
 173         default:
 174                 EFSYS_ASSERT(0);
 175                 rc = ENOTSUP;
 176                 goto fail1;
 177         }
 178
 179         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 180
 181         if ((rc = eevop->eevo_init(enp)) != 0)
 182                 goto fail2;
 183
 184         enp->en_eevop = eevop;
 185         enp->en_mod_flags |= EFX_MOD_EV;
 186         return (0);
 187
 188 fail2:
 189         EFSYS_PROBE(fail2);
 190
 191 fail1:
 192         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 193
 194         enp->en_eevop = NULL;
 195         enp->en_mod_flags &= ~EFX_MOD_EV;
 196         return (rc);
 197 }
 198
 199                 void
 200 efx_ev_fini(
 201         __in    efx_nic_t *enp)
 202 {
 203         const efx_ev_ops_t *eevop = enp->en_eevop;
 204
 205         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 206         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 207         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 208         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
 209         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
 210         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 211
 212         eevop->eevo_fini(enp);
 213
 214         enp->en_eevop = NULL;
 215         enp->en_mod_flags &= ~EFX_MOD_EV;
 216 }
 217
 218
 219         __checkReturn   efx_rc_t
 220 efx_ev_qcreate(
 221         __in            efx_nic_t *enp,
 222         __in            unsigned int index,
 223         __in            efsys_mem_t *esmp,
 224         __in            size_t n,
 225         __in            uint32_t id,
 226         __in            uint32_t us,
 227         __in            uint32_t flags,
 228         __deref_out     efx_evq_t **eepp)
 229 {
 230         const efx_ev_ops_t *eevop = enp->en_eevop;
 231         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 232         efx_evq_t *eep;
 233         efx_rc_t rc;
 234
 235         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 236         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 237
 238         EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
 239
 240         switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) {
 241         case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT:
 242                 break;
 243         case EFX_EVQ_FLAGS_NOTIFY_DISABLED:
 244                 if (us != 0) {
 245                         rc = EINVAL;
 246                         goto fail1;
 247                 }
 248                 break;
 249         default:
 250                 rc = EINVAL;
 251                 goto fail2;
 252         }
 253
 254         /* Allocate an EVQ object */
 255         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
 256         if (eep == NULL) {
 257                 rc = ENOMEM;
 258                 goto fail3;
 259         }
 260
 261         eep->ee_magic = EFX_EVQ_MAGIC;
 262         eep->ee_enp = enp;
 263         eep->ee_index = index;
 264         eep->ee_mask = n - 1;
 265         eep->ee_flags = flags;
 266         eep->ee_esmp = esmp;
 267
 268         /*
 269          * Set outputs before the queue is created because interrupts may be
 270          * raised for events immediately after the queue is created, before the
 271          * function call below returns. See bug58606.
 272          *
 273          * The eepp pointer passed in by the client must therefore point to data
 274          * shared with the client's event processing context.
 275          */
 276         enp->en_ev_qcount++;
 277         *eepp = eep;
 278
 279         if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, us, flags,
 280             eep)) != 0)
 281                 goto fail4;
 282
 283         return (0);
 284
 285 fail4:
 286         EFSYS_PROBE(fail4);
 287
 288         *eepp = NULL;
 289         enp->en_ev_qcount--;
 290         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 291 fail3:
 292         EFSYS_PROBE(fail3);
 293 fail2:
 294         EFSYS_PROBE(fail2);
 295 fail1:
 296         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 297         return (rc);
 298 }
 299
 300                 void
 301 efx_ev_qdestroy(
 302         __in    efx_evq_t *eep)
 303 {
 304         efx_nic_t *enp = eep->ee_enp;
 305         const efx_ev_ops_t *eevop = enp->en_eevop;
 306
 307         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 308
 309         EFSYS_ASSERT(enp->en_ev_qcount != 0);
 310         --enp->en_ev_qcount;
 311
 312         eevop->eevo_qdestroy(eep);
 313
 314         /* Free the EVQ object */
 315         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 316 }
 317
 318         __checkReturn   efx_rc_t
 319 efx_ev_qprime(
 320         __in            efx_evq_t *eep,
 321         __in            unsigned int count)
 322 {
 323         efx_nic_t *enp = eep->ee_enp;
 324         const efx_ev_ops_t *eevop = enp->en_eevop;
 325         efx_rc_t rc;
 326
 327         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 328
 329         if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
 330                 rc = EINVAL;
 331                 goto fail1;
 332         }
 333
 334         if ((rc = eevop->eevo_qprime(eep, count)) != 0)
 335                 goto fail2;
 336
 337         return (0);
 338
 339 fail2:
 340         EFSYS_PROBE(fail2);
 341 fail1:
 342         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 343         return (rc);
 344 }
 345
 346         __checkReturn   boolean_t
 347 efx_ev_qpending(
 348         __in            efx_evq_t *eep,
 349         __in            unsigned int count)
 350 {
 351         size_t offset;
 352         efx_qword_t qword;
 353
 354         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 355
 356         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 357         EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
 358
 359         return (EFX_EV_PRESENT(qword));
 360 }
 361
 362 #if EFSYS_OPT_EV_PREFETCH
 363
 364                         void
 365 efx_ev_qprefetch(
 366         __in            efx_evq_t *eep,
 367         __in            unsigned int count)
 368 {
 369         unsigned int offset;
 370
 371         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 372
 373         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 374         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 375 }
 376
 377 #endif  /* EFSYS_OPT_EV_PREFETCH */
 378
 379 #define EFX_EV_BATCH    8
 380
 381                         void
 382 efx_ev_qpoll(
 383         __in            efx_evq_t *eep,
 384         __inout         unsigned int *countp,
 385         __in            const efx_ev_callbacks_t *eecp,
 386         __in_opt        void *arg)
 387 {
 388         efx_qword_t ev[EFX_EV_BATCH];
 389         unsigned int batch;
 390         unsigned int total;
 391         unsigned int count;
 392         unsigned int index;
 393         size_t offset;
 394
 395         /* Ensure events codes match for EF10 (Huntington/Medford) and Siena */
 396         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
 397         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
 398
 399         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
 400         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
 401         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
 402         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
 403             FSE_AZ_EV_CODE_DRV_GEN_EV);
 404 #if EFSYS_OPT_MCDI
 405         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
 406             FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
 407 #endif
 408
 409         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 410         EFSYS_ASSERT(countp != NULL);
 411         EFSYS_ASSERT(eecp != NULL);
 412
 413         count = *countp;
 414         do {
 415                 /* Read up until the end of the batch period */
 416                 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
 417                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 418                 for (total = 0; total < batch; ++total) {
 419                         EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
 420
 421                         if (!EFX_EV_PRESENT(ev[total]))
 422                                 break;
 423
 424                         EFSYS_PROBE3(event, unsigned int, eep->ee_index,
 425                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
 426                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
 427
 428                         offset += sizeof (efx_qword_t);
 429                 }
 430
 431 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
 432                 /*
 433                  * Prefetch the next batch when we get within PREFETCH_PERIOD
 434                  * of a completed batch. If the batch is smaller, then prefetch
 435                  * immediately.
 436                  */
 437                 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
 438                         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 439 #endif  /* EFSYS_OPT_EV_PREFETCH */
 440
 441                 /* Process the batch of events */
 442                 for (index = 0; index < total; ++index) {
 443                         boolean_t should_abort;
 444                         uint32_t code;
 445
 446 #if EFSYS_OPT_EV_PREFETCH
 447                         /* Prefetch if we've now reached the batch period */
 448                         if (total == batch &&
 449                             index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
 450                                 offset = (count + batch) & eep->ee_mask;
 451                                 offset *= sizeof (efx_qword_t);
 452
 453                                 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 454                         }
 455 #endif  /* EFSYS_OPT_EV_PREFETCH */
 456
 457                         EFX_EV_QSTAT_INCR(eep, EV_ALL);
 458
 459                         code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
 460                         switch (code) {
 461                         case FSE_AZ_EV_CODE_RX_EV:
 462                                 should_abort = eep->ee_rx(eep,
 463                                     &(ev[index]), eecp, arg);
 464                                 break;
 465                         case FSE_AZ_EV_CODE_TX_EV:
 466                                 should_abort = eep->ee_tx(eep,
 467                                     &(ev[index]), eecp, arg);
 468                                 break;
 469                         case FSE_AZ_EV_CODE_DRIVER_EV:
 470                                 should_abort = eep->ee_driver(eep,
 471                                     &(ev[index]), eecp, arg);
 472                                 break;
 473                         case FSE_AZ_EV_CODE_DRV_GEN_EV:
 474                                 should_abort = eep->ee_drv_gen(eep,
 475                                     &(ev[index]), eecp, arg);
 476                                 break;
 477 #if EFSYS_OPT_MCDI
 478                         case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
 479                                 should_abort = eep->ee_mcdi(eep,
 480                                     &(ev[index]), eecp, arg);
 481                                 break;
 482 #endif
 483                         case FSE_AZ_EV_CODE_GLOBAL_EV:
 484                                 if (eep->ee_global) {
 485                                         should_abort = eep->ee_global(eep,
 486                                             &(ev[index]), eecp, arg);
 487                                         break;
 488                                 }
 489                                 /* else fallthrough */
 490                         default:
 491                                 EFSYS_PROBE3(bad_event,
 492                                     unsigned int, eep->ee_index,
 493                                     uint32_t,
 494                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
 495                                     uint32_t,
 496                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
 497
 498                                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 499                                 (void) eecp->eec_exception(arg,
 500                                         EFX_EXCEPTION_EV_ERROR, code);
 501                                 should_abort = B_TRUE;
 502                         }
 503                         if (should_abort) {
 504                                 /* Ignore subsequent events */
 505                                 total = index + 1;
 506                                 break;
 507                         }
 508                 }
 509
 510                 /*
 511                  * Now that the hardware has most likely moved onto dma'ing
 512                  * into the next cache line, clear the processed events. Take
 513                  * care to only clear out events that we've processed
 514                  */
 515                 EFX_SET_QWORD(ev[0]);
 516                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 517                 for (index = 0; index < total; ++index) {
 518                         EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
 519                         offset += sizeof (efx_qword_t);
 520                 }
 521
 522                 count += total;
 523
 524         } while (total == batch);
 525
 526         *countp = count;
 527 }
 528
 529                         void
 530 efx_ev_qpost(
 531         __in    efx_evq_t *eep,
 532         __in    uint16_t data)
 533 {
 534         efx_nic_t *enp = eep->ee_enp;
 535         const efx_ev_ops_t *eevop = enp->en_eevop;
 536
 537         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 538
 539         EFSYS_ASSERT(eevop != NULL &&
 540             eevop->eevo_qpost != NULL);
 541
 542         eevop->eevo_qpost(eep, data);
 543 }
 544
 545         __checkReturn   efx_rc_t
 546 efx_ev_usecs_to_ticks(
 547         __in            efx_nic_t *enp,
 548         __in            unsigned int us,
 549         __out           unsigned int *ticksp)
 550 {
 551         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 552         unsigned int ticks;
 553
 554         /* Convert microseconds to a timer tick count */
 555         if (us == 0)
 556                 ticks = 0;
 557         else if (us * 1000 < encp->enc_evq_timer_quantum_ns)
 558                 ticks = 1;      /* Never round down to zero */
 559         else
 560                 ticks = us * 1000 / encp->enc_evq_timer_quantum_ns;
 561
 562         *ticksp = ticks;
 563         return (0);
 564 }
 565
 566         __checkReturn   efx_rc_t
 567 efx_ev_qmoderate(
 568         __in            efx_evq_t *eep,
 569         __in            unsigned int us)
 570 {
 571         efx_nic_t *enp = eep->ee_enp;
 572         const efx_ev_ops_t *eevop = enp->en_eevop;
 573         efx_rc_t rc;
 574
 575         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 576
 577         if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
 578             EFX_EVQ_FLAGS_NOTIFY_DISABLED) {
 579                 rc = EINVAL;
 580                 goto fail1;
 581         }
 582
 583         if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
 584                 goto fail2;
 585
 586         return (0);
 587
 588 fail2:
 589         EFSYS_PROBE(fail2);
 590 fail1:
 591         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 592         return (rc);
 593 }
 594
 595 #if EFSYS_OPT_QSTATS
 596                                         void
 597 efx_ev_qstats_update(
 598         __in                            efx_evq_t *eep,
 599         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
 600
 601 {       efx_nic_t *enp = eep->ee_enp;
 602         const efx_ev_ops_t *eevop = enp->en_eevop;
 603
 604         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 605
 606         eevop->eevo_qstats_update(eep, stat);
 607 }
 608
 609 #endif  /* EFSYS_OPT_QSTATS */
 610
 611 #if EFSYS_OPT_SIENA
 612
 613 static  __checkReturn   efx_rc_t
 614 siena_ev_init(
 615         __in            efx_nic_t *enp)
 616 {
 617         efx_oword_t oword;
 618
 619         /*
 620          * Program the event queue for receive and transmit queue
 621          * flush events.
 622          */
 623         EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
 624         EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
 625         EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
 626
 627         return (0);
 628
 629 }
 630
 631 static  __checkReturn   boolean_t
 632 siena_ev_rx_not_ok(
 633         __in            efx_evq_t *eep,
 634         __in            efx_qword_t *eqp,
 635         __in            uint32_t label,
 636         __in            uint32_t id,
 637         __inout         uint16_t *flagsp)
 638 {
 639         boolean_t ignore = B_FALSE;
 640
 641         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
 642                 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
 643                 EFSYS_PROBE(tobe_disc);
 644                 /*
 645                  * Assume this is a unicast address mismatch, unless below
 646                  * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
 647                  * EV_RX_PAUSE_FRM_ERR is set.
 648                  */
 649                 (*flagsp) |= EFX_ADDR_MISMATCH;
 650         }
 651
 652         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
 653                 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
 654                 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
 655                 (*flagsp) |= EFX_DISCARD;
 656
 657 #if EFSYS_OPT_RX_SCATTER
 658                 /*
 659                  * Lookout for payload queue ran dry errors and ignore them.
 660                  *
 661                  * Sadly for the header/data split cases, the descriptor
 662                  * pointer in this event refers to the header queue and
 663                  * therefore cannot be easily detected as duplicate.
 664                  * So we drop these and rely on the receive processing seeing
 665                  * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
 666                  * the partially received packet.
 667                  */
 668                 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
 669                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
 670                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
 671                         ignore = B_TRUE;
 672 #endif  /* EFSYS_OPT_RX_SCATTER */
 673         }
 674
 675         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
 676                 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
 677                 EFSYS_PROBE(crc_err);
 678                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 679                 (*flagsp) |= EFX_DISCARD;
 680         }
 681
 682         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
 683                 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
 684                 EFSYS_PROBE(pause_frm_err);
 685                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 686                 (*flagsp) |= EFX_DISCARD;
 687         }
 688
 689         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
 690                 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
 691                 EFSYS_PROBE(owner_id_err);
 692                 (*flagsp) |= EFX_DISCARD;
 693         }
 694
 695         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
 696                 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
 697                 EFSYS_PROBE(ipv4_err);
 698                 (*flagsp) &= ~EFX_CKSUM_IPV4;
 699         }
 700
 701         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
 702                 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
 703                 EFSYS_PROBE(udp_chk_err);
 704                 (*flagsp) &= ~EFX_CKSUM_TCPUDP;
 705         }
 706
 707         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
 708                 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
 709
 710                 /*
 711                  * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
 712                  * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
 713                  * condition.
 714                  */
 715                 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
 716         }
 717
 718         return (ignore);
 719 }
 720
 721 static  __checkReturn   boolean_t
 722 siena_ev_rx(
 723         __in            efx_evq_t *eep,
 724         __in            efx_qword_t *eqp,
 725         __in            const efx_ev_callbacks_t *eecp,
 726         __in_opt        void *arg)
 727 {
 728         uint32_t id;
 729         uint32_t size;
 730         uint32_t label;
 731         boolean_t ok;
 732 #if EFSYS_OPT_RX_SCATTER
 733         boolean_t sop;
 734         boolean_t jumbo_cont;
 735 #endif  /* EFSYS_OPT_RX_SCATTER */
 736         uint32_t hdr_type;
 737         boolean_t is_v6;
 738         uint16_t flags;
 739         boolean_t ignore;
 740         boolean_t should_abort;
 741
 742         EFX_EV_QSTAT_INCR(eep, EV_RX);
 743
 744         /* Basic packet information */
 745         id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
 746         size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
 747         label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
 748         ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
 749
 750 #if EFSYS_OPT_RX_SCATTER
 751         sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
 752         jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
 753 #endif  /* EFSYS_OPT_RX_SCATTER */
 754
 755         hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
 756
 757         is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
 758
 759         /*
 760          * If packet is marked as OK and packet type is TCP/IP or
 761          * UDP/IP or other IP, then we can rely on the hardware checksums.
 762          */
 763         switch (hdr_type) {
 764         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
 765                 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
 766                 if (is_v6) {
 767                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
 768                         flags |= EFX_PKT_IPV6;
 769                 } else {
 770                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
 771                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 772                 }
 773                 break;
 774
 775         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
 776                 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
 777                 if (is_v6) {
 778                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
 779                         flags |= EFX_PKT_IPV6;
 780                 } else {
 781                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
 782                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 783                 }
 784                 break;
 785
 786         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
 787                 if (is_v6) {
 788                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
 789                         flags = EFX_PKT_IPV6;
 790                 } else {
 791                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
 792                         flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 793                 }
 794                 break;
 795
 796         case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
 797                 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
 798                 flags = 0;
 799                 break;
 800
 801         default:
 802                 EFSYS_ASSERT(B_FALSE);
 803                 flags = 0;
 804                 break;
 805         }
 806
 807 #if EFSYS_OPT_RX_SCATTER
 808         /* Report scatter and header/lookahead split buffer flags */
 809         if (sop)
 810                 flags |= EFX_PKT_START;
 811         if (jumbo_cont)
 812                 flags |= EFX_PKT_CONT;
 813 #endif  /* EFSYS_OPT_RX_SCATTER */
 814
 815         /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
 816         if (!ok) {
 817                 ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
 818                 if (ignore) {
 819                         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 820                             uint32_t, size, uint16_t, flags);
 821
 822                         return (B_FALSE);
 823                 }
 824         }
 825
 826         /* If we're not discarding the packet then it is ok */
 827         if (~flags & EFX_DISCARD)
 828                 EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
 829
 830         /* Detect multicast packets that didn't match the filter */
 831         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
 832                 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
 833
 834                 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
 835                         EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
 836                 } else {
 837                         EFSYS_PROBE(mcast_mismatch);
 838                         flags |= EFX_ADDR_MISMATCH;
 839                 }
 840         } else {
 841                 flags |= EFX_PKT_UNICAST;
 842         }
 843
 844         /*
 845          * The packet parser in Siena can abort parsing packets under
 846          * certain error conditions, setting the PKT_NOT_PARSED bit
 847          * (which clears PKT_OK). If this is set, then don't trust
 848          * the PKT_TYPE field.
 849          */
 850         if (!ok) {
 851                 uint32_t parse_err;
 852
 853                 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
 854                 if (parse_err != 0)
 855                         flags |= EFX_CHECK_VLAN;
 856         }
 857
 858         if (~flags & EFX_CHECK_VLAN) {
 859                 uint32_t pkt_type;
 860
 861                 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
 862                 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
 863                         flags |= EFX_PKT_VLAN_TAGGED;
 864         }
 865
 866         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 867             uint32_t, size, uint16_t, flags);
 868
 869         EFSYS_ASSERT(eecp->eec_rx != NULL);
 870         should_abort = eecp->eec_rx(arg, label, id, size, flags);
 871
 872         return (should_abort);
 873 }
 874
 875 static  __checkReturn   boolean_t
 876 siena_ev_tx(
 877         __in            efx_evq_t *eep,
 878         __in            efx_qword_t *eqp,
 879         __in            const efx_ev_callbacks_t *eecp,
 880         __in_opt        void *arg)
 881 {
 882         uint32_t id;
 883         uint32_t label;
 884         boolean_t should_abort;
 885
 886         EFX_EV_QSTAT_INCR(eep, EV_TX);
 887
 888         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
 889             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
 890             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
 891             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
 892
 893                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
 894                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
 895
 896                 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
 897
 898                 EFSYS_ASSERT(eecp->eec_tx != NULL);
 899                 should_abort = eecp->eec_tx(arg, label, id);
 900
 901                 return (should_abort);
 902         }
 903
 904         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
 905                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 906                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 907                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 908
 909         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
 910                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
 911
 912         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
 913                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
 914
 915         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
 916                 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
 917
 918         EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
 919         return (B_FALSE);
 920 }
 921
 922 static  __checkReturn   boolean_t
 923 siena_ev_global(
 924         __in            efx_evq_t *eep,
 925         __in            efx_qword_t *eqp,
 926         __in            const efx_ev_callbacks_t *eecp,
 927         __in_opt        void *arg)
 928 {
 929         _NOTE(ARGUNUSED(eqp, eecp, arg))
 930
 931         EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
 932
 933         return (B_FALSE);
 934 }
 935
 936 static  __checkReturn   boolean_t
 937 siena_ev_driver(
 938         __in            efx_evq_t *eep,
 939         __in            efx_qword_t *eqp,
 940         __in            const efx_ev_callbacks_t *eecp,
 941         __in_opt        void *arg)
 942 {
 943         boolean_t should_abort;
 944
 945         EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
 946         should_abort = B_FALSE;
 947
 948         switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
 949         case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
 950                 uint32_t txq_index;
 951
 952                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
 953
 954                 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 955
 956                 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
 957
 958                 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
 959                 should_abort = eecp->eec_txq_flush_done(arg, txq_index);
 960
 961                 break;
 962         }
 963         case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
 964                 uint32_t rxq_index;
 965                 uint32_t failed;
 966
 967                 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
 968                 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
 969
 970                 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
 971                 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
 972
 973                 if (failed) {
 974                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
 975
 976                         EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
 977
 978                         should_abort = eecp->eec_rxq_flush_failed(arg,
 979                                                                     rxq_index);
 980                 } else {
 981                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
 982
 983                         EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
 984
 985                         should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
 986                 }
 987
 988                 break;
 989         }
 990         case FSE_AZ_EVQ_INIT_DONE_EV:
 991                 EFSYS_ASSERT(eecp->eec_initialized != NULL);
 992                 should_abort = eecp->eec_initialized(arg);
 993
 994                 break;
 995
 996         case FSE_AZ_EVQ_NOT_EN_EV:
 997                 EFSYS_PROBE(evq_not_en);
 998                 break;
 999
1000         case FSE_AZ_SRM_UPD_DONE_EV: {
1001                 uint32_t code;
1002
1003                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
1004
1005                 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1006
1007                 EFSYS_ASSERT(eecp->eec_sram != NULL);
1008                 should_abort = eecp->eec_sram(arg, code);
1009
1010                 break;
1011         }
1012         case FSE_AZ_WAKE_UP_EV: {
1013                 uint32_t id;
1014
1015                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1016
1017                 EFSYS_ASSERT(eecp->eec_wake_up != NULL);
1018                 should_abort = eecp->eec_wake_up(arg, id);
1019
1020                 break;
1021         }
1022         case FSE_AZ_TX_PKT_NON_TCP_UDP:
1023                 EFSYS_PROBE(tx_pkt_non_tcp_udp);
1024                 break;
1025
1026         case FSE_AZ_TIMER_EV: {
1027                 uint32_t id;
1028
1029                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
1030
1031                 EFSYS_ASSERT(eecp->eec_timer != NULL);
1032                 should_abort = eecp->eec_timer(arg, id);
1033
1034                 break;
1035         }
1036         case FSE_AZ_RX_DSC_ERROR_EV:
1037                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
1038
1039                 EFSYS_PROBE(rx_dsc_error);
1040
1041                 EFSYS_ASSERT(eecp->eec_exception != NULL);
1042                 should_abort = eecp->eec_exception(arg,
1043                         EFX_EXCEPTION_RX_DSC_ERROR, 0);
1044
1045                 break;
1046
1047         case FSE_AZ_TX_DSC_ERROR_EV:
1048                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
1049
1050                 EFSYS_PROBE(tx_dsc_error);
1051
1052                 EFSYS_ASSERT(eecp->eec_exception != NULL);
1053                 should_abort = eecp->eec_exception(arg,
1054                         EFX_EXCEPTION_TX_DSC_ERROR, 0);
1055
1056                 break;
1057
1058         default:
1059                 break;
1060         }
1061
1062         return (should_abort);
1063 }
1064
1065 static  __checkReturn   boolean_t
1066 siena_ev_drv_gen(
1067         __in            efx_evq_t *eep,
1068         __in            efx_qword_t *eqp,
1069         __in            const efx_ev_callbacks_t *eecp,
1070         __in_opt        void *arg)
1071 {
1072         uint32_t data;
1073         boolean_t should_abort;
1074
1075         EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
1076
1077         data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
1078         if (data >= ((uint32_t)1 << 16)) {
1079                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
1080                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
1081                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
1082                 return (B_TRUE);
1083         }
1084
1085         EFSYS_ASSERT(eecp->eec_software != NULL);
1086         should_abort = eecp->eec_software(arg, (uint16_t)data);
1087
1088         return (should_abort);
1089 }
1090
1091 #if EFSYS_OPT_MCDI
1092
1093 static  __checkReturn   boolean_t
1094 siena_ev_mcdi(
1095         __in            efx_evq_t *eep,
1096         __in            efx_qword_t *eqp,
1097         __in            const efx_ev_callbacks_t *eecp,
1098         __in_opt        void *arg)
1099 {
1100         efx_nic_t *enp = eep->ee_enp;
1101         unsigned int code;
1102         boolean_t should_abort = B_FALSE;
1103
1104         EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
1105
1106         if (enp->en_family != EFX_FAMILY_SIENA)
1107                 goto out;
1108
1109         EFSYS_ASSERT(eecp->eec_link_change != NULL);
1110         EFSYS_ASSERT(eecp->eec_exception != NULL);
1111 #if EFSYS_OPT_MON_STATS
1112         EFSYS_ASSERT(eecp->eec_monitor != NULL);
1113 #endif
1114
1115         EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
1116
1117         code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
1118         switch (code) {
1119         case MCDI_EVENT_CODE_BADSSERT:
1120                 efx_mcdi_ev_death(enp, EINTR);
1121                 break;
1122
1123         case MCDI_EVENT_CODE_CMDDONE:
1124                 efx_mcdi_ev_cpl(enp,
1125                     MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
1126                     MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
1127                     MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
1128                 break;
1129
1130         case MCDI_EVENT_CODE_LINKCHANGE: {
1131                 efx_link_mode_t link_mode;
1132
1133                 siena_phy_link_ev(enp, eqp, &link_mode);
1134                 should_abort = eecp->eec_link_change(arg, link_mode);
1135                 break;
1136         }
1137         case MCDI_EVENT_CODE_SENSOREVT: {
1138 #if EFSYS_OPT_MON_STATS
1139                 efx_mon_stat_t id;
1140                 efx_mon_stat_value_t value;
1141                 efx_rc_t rc;
1142
1143                 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
1144                         should_abort = eecp->eec_monitor(arg, id, value);
1145                 else if (rc == ENOTSUP) {
1146                         should_abort = eecp->eec_exception(arg,
1147                                 EFX_EXCEPTION_UNKNOWN_SENSOREVT,
1148                                 MCDI_EV_FIELD(eqp, DATA));
1149                 } else
1150                         EFSYS_ASSERT(rc == ENODEV);     /* Wrong port */
1151 #else
1152                 should_abort = B_FALSE;
1153 #endif
1154                 break;
1155         }
1156         case MCDI_EVENT_CODE_SCHEDERR:
1157                 /* Informational only */
1158                 break;
1159
1160         case MCDI_EVENT_CODE_REBOOT:
1161                 efx_mcdi_ev_death(enp, EIO);
1162                 break;
1163
1164         case MCDI_EVENT_CODE_MAC_STATS_DMA:
1165 #if EFSYS_OPT_MAC_STATS
1166                 if (eecp->eec_mac_stats != NULL) {
1167                         eecp->eec_mac_stats(arg,
1168                             MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
1169                 }
1170 #endif
1171                 break;
1172
1173         case MCDI_EVENT_CODE_FWALERT: {
1174                 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
1175
1176                 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
1177                         should_abort = eecp->eec_exception(arg,
1178                                 EFX_EXCEPTION_FWALERT_SRAM,
1179                                 MCDI_EV_FIELD(eqp, FWALERT_DATA));
1180                 else
1181                         should_abort = eecp->eec_exception(arg,
1182                                 EFX_EXCEPTION_UNKNOWN_FWALERT,
1183                                 MCDI_EV_FIELD(eqp, DATA));
1184                 break;
1185         }
1186
1187         default:
1188                 EFSYS_PROBE1(mc_pcol_error, int, code);
1189                 break;
1190         }
1191
1192 out:
1193         return (should_abort);
1194 }
1195
1196 #endif  /* EFSYS_OPT_MCDI */
1197
1198 static  __checkReturn   efx_rc_t
1199 siena_ev_qprime(
1200         __in            efx_evq_t *eep,
1201         __in            unsigned int count)
1202 {
1203         efx_nic_t *enp = eep->ee_enp;
1204         uint32_t rptr;
1205         efx_dword_t dword;
1206
1207         rptr = count & eep->ee_mask;
1208
1209         EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
1210
1211         EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
1212                             &dword, B_FALSE);
1213
1214         return (0);
1215 }
1216
1217 static          void
1218 siena_ev_qpost(
1219         __in    efx_evq_t *eep,
1220         __in    uint16_t data)
1221 {
1222         efx_nic_t *enp = eep->ee_enp;
1223         efx_qword_t ev;
1224         efx_oword_t oword;
1225
1226         EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
1227             FSF_AZ_EV_DATA_DW0, (uint32_t)data);
1228
1229         EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
1230             EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
1231             EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
1232
1233         EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
1234 }
1235
1236 static  __checkReturn   efx_rc_t
1237 siena_ev_qmoderate(
1238         __in            efx_evq_t *eep,
1239         __in            unsigned int us)
1240 {
1241         efx_nic_t *enp = eep->ee_enp;
1242         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1243         unsigned int locked;
1244         efx_dword_t dword;
1245         efx_rc_t rc;
1246
1247         if (us > encp->enc_evq_timer_max_us) {
1248                 rc = EINVAL;
1249                 goto fail1;
1250         }
1251
1252         /* If the value is zero then disable the timer */
1253         if (us == 0) {
1254                 EFX_POPULATE_DWORD_2(dword,
1255                     FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
1256                     FRF_CZ_TC_TIMER_VAL, 0);
1257         } else {
1258                 unsigned int ticks;
1259
1260                 if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0)
1261                         goto fail2;
1262
1263                 EFSYS_ASSERT(ticks > 0);
1264                 EFX_POPULATE_DWORD_2(dword,
1265                     FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
1266                     FRF_CZ_TC_TIMER_VAL, ticks - 1);
1267         }
1268
1269         locked = (eep->ee_index == 0) ? 1 : 0;
1270
1271         EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
1272             eep->ee_index, &dword, locked);
1273
1274         return (0);
1275
1276 fail2:
1277         EFSYS_PROBE(fail2);
1278 fail1:
1279         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1280
1281         return (rc);
1282 }
1283
1284 static  __checkReturn   efx_rc_t
1285 siena_ev_qcreate(
1286         __in            efx_nic_t *enp,
1287         __in            unsigned int index,
1288         __in            efsys_mem_t *esmp,
1289         __in            size_t n,
1290         __in            uint32_t id,
1291         __in            uint32_t us,
1292         __in            uint32_t flags,
1293         __in            efx_evq_t *eep)
1294 {
1295         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1296         uint32_t size;
1297         efx_oword_t oword;
1298         efx_rc_t rc;
1299         boolean_t notify_mode;
1300
1301         _NOTE(ARGUNUSED(esmp))
1302
1303         EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS));
1304         EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS));
1305
1306         if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) {
1307                 rc = EINVAL;
1308                 goto fail1;
1309         }
1310         if (index >= encp->enc_evq_limit) {
1311                 rc = EINVAL;
1312                 goto fail2;
1313         }
1314 #if EFSYS_OPT_RX_SCALE
1315         if (enp->en_intr.ei_type == EFX_INTR_LINE &&
1316             index >= EFX_MAXRSS_LEGACY) {
1317                 rc = EINVAL;
1318                 goto fail3;
1319         }
1320 #endif
1321         for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
1322             size++)
1323                 if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
1324                         break;
1325         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1326                 rc = EINVAL;
1327                 goto fail4;
1328         }
1329
1330         /* Set up the handler table */
1331         eep->ee_rx      = siena_ev_rx;
1332         eep->ee_tx      = siena_ev_tx;
1333         eep->ee_driver  = siena_ev_driver;
1334         eep->ee_global  = siena_ev_global;
1335         eep->ee_drv_gen = siena_ev_drv_gen;
1336 #if EFSYS_OPT_MCDI
1337         eep->ee_mcdi    = siena_ev_mcdi;
1338 #endif  /* EFSYS_OPT_MCDI */
1339
1340         notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) !=
1341             EFX_EVQ_FLAGS_NOTIFY_INTERRUPT);
1342
1343         /* Set up the new event queue */
1344         EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1,
1345             FRF_CZ_HOST_NOTIFY_MODE, notify_mode,
1346             FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
1347         EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
1348
1349         EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
1350             FRF_AZ_EVQ_BUF_BASE_ID, id);
1351
1352         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
1353
1354         /* Set initial interrupt moderation */
1355         siena_ev_qmoderate(eep, us);
1356
1357         return (0);
1358
1359 fail4:
1360         EFSYS_PROBE(fail4);
1361 #if EFSYS_OPT_RX_SCALE
1362 fail3:
1363         EFSYS_PROBE(fail3);
1364 #endif
1365 fail2:
1366         EFSYS_PROBE(fail2);
1367 fail1:
1368         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1369
1370         return (rc);
1371 }
1372
1373 #endif /* EFSYS_OPT_SIENA */
1374
1375 #if EFSYS_OPT_QSTATS
1376 #if EFSYS_OPT_NAMES
1377 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock c0f3bc5083b40532 */
1378 static const char * const __efx_ev_qstat_name[] = {
1379         "all",
1380         "rx",
1381         "rx_ok",
1382         "rx_frm_trunc",
1383         "rx_tobe_disc",
1384         "rx_pause_frm_err",
1385         "rx_buf_owner_id_err",
1386         "rx_ipv4_hdr_chksum_err",
1387         "rx_tcp_udp_chksum_err",
1388         "rx_eth_crc_err",
1389         "rx_ip_frag_err",
1390         "rx_mcast_pkt",
1391         "rx_mcast_hash_match",
1392         "rx_tcp_ipv4",
1393         "rx_tcp_ipv6",
1394         "rx_udp_ipv4",
1395         "rx_udp_ipv6",
1396         "rx_other_ipv4",
1397         "rx_other_ipv6",
1398         "rx_non_ip",
1399         "rx_batch",
1400         "tx",
1401         "tx_wq_ff_full",
1402         "tx_pkt_err",
1403         "tx_pkt_too_big",
1404         "tx_unexpected",
1405         "global",
1406         "global_mnt",
1407         "driver",
1408         "driver_srm_upd_done",
1409         "driver_tx_descq_fls_done",
1410         "driver_rx_descq_fls_done",
1411         "driver_rx_descq_fls_failed",
1412         "driver_rx_dsc_error",
1413         "driver_tx_dsc_error",
1414         "drv_gen",
1415         "mcdi_response",
1416 };
1417 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1418
1419                 const char *
1420 efx_ev_qstat_name(
1421         __in    efx_nic_t *enp,
1422         __in    unsigned int id)
1423 {
1424         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1425         EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1426
1427         return (__efx_ev_qstat_name[id]);
1428 }
1429 #endif  /* EFSYS_OPT_NAMES */
1430 #endif  /* EFSYS_OPT_QSTATS */
1431
1432 #if EFSYS_OPT_SIENA
1433
1434 #if EFSYS_OPT_QSTATS
1435 static                                  void
1436 siena_ev_qstats_update(
1437         __in                            efx_evq_t *eep,
1438         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
1439 {
1440         unsigned int id;
1441
1442         for (id = 0; id < EV_NQSTATS; id++) {
1443                 efsys_stat_t *essp = &stat[id];
1444
1445                 EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1446                 eep->ee_stat[id] = 0;
1447         }
1448 }
1449 #endif  /* EFSYS_OPT_QSTATS */
1450
1451 static          void
1452 siena_ev_qdestroy(
1453         __in    efx_evq_t *eep)
1454 {
1455         efx_nic_t *enp = eep->ee_enp;
1456         efx_oword_t oword;
1457
1458         /* Purge event queue */
1459         EFX_ZERO_OWORD(oword);
1460
1461         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1462             eep->ee_index, &oword, B_TRUE);
1463
1464         EFX_ZERO_OWORD(oword);
1465         EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE);
1466 }
1467
1468 static          void
1469 siena_ev_fini(
1470         __in    efx_nic_t *enp)
1471 {
1472         _NOTE(ARGUNUSED(enp))
1473 }
1474
1475 #endif /* EFSYS_OPT_SIENA */