sys/dev/sfxge/common/efx_ev.c

   1 /*-
   2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "efsys.h"
  35 #include "efx.h"
  36 #include "efx_types.h"
  37 #include "efx_regs.h"
  38 #include "efx_impl.h"
  39 #include "mcdi_mon.h"
  40
  41 #if EFSYS_OPT_QSTATS
  42 #define EFX_EV_QSTAT_INCR(_eep, _stat)                                  \
  43         do {                                                            \
  44                 (_eep)->ee_stat[_stat]++;                               \
  45         _NOTE(CONSTANTCONDITION)                                        \
  46         } while (B_FALSE)
  47 #else
  48 #define EFX_EV_QSTAT_INCR(_eep, _stat)
  49 #endif
  50
  51 #define EFX_EV_PRESENT(_qword)                                          \
  52         (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&        \
  53         EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
  54
  55
  56
  57 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
  58
  59 static  __checkReturn   int
  60 falconsiena_ev_init(
  61         __in            efx_nic_t *enp);
  62
  63 static                  void
  64 falconsiena_ev_fini(
  65         __in            efx_nic_t *enp);
  66
  67 static  __checkReturn   int
  68 falconsiena_ev_qcreate(
  69         __in            efx_nic_t *enp,
  70         __in            unsigned int index,
  71         __in            efsys_mem_t *esmp,
  72         __in            size_t n,
  73         __in            uint32_t id,
  74         __in            efx_evq_t *eep);
  75
  76 static                  void
  77 falconsiena_ev_qdestroy(
  78         __in            efx_evq_t *eep);
  79
  80 static  __checkReturn   int
  81 falconsiena_ev_qprime(
  82         __in            efx_evq_t *eep,
  83         __in            unsigned int count);
  84
  85 static                  void
  86 falconsiena_ev_qpoll(
  87         __in            efx_evq_t *eep,
  88         __inout         unsigned int *countp,
  89         __in            const efx_ev_callbacks_t *eecp,
  90         __in_opt        void *arg);
  91
  92 static                  void
  93 falconsiena_ev_qpost(
  94         __in    efx_evq_t *eep,
  95         __in    uint16_t data);
  96
  97 static  __checkReturn   int
  98 falconsiena_ev_qmoderate(
  99         __in            efx_evq_t *eep,
 100         __in            unsigned int us);
 101
 102 #if EFSYS_OPT_QSTATS
 103 static                  void
 104 falconsiena_ev_qstats_update(
 105         __in                            efx_evq_t *eep,
 106         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat);
 107
 108 #endif
 109
 110 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
 111
 112 #if EFSYS_OPT_FALCON
 113 static efx_ev_ops_t     __efx_ev_falcon_ops = {
 114         falconsiena_ev_init,                    /* eevo_init */
 115         falconsiena_ev_fini,                    /* eevo_fini */
 116         falconsiena_ev_qcreate,                 /* eevo_qcreate */
 117         falconsiena_ev_qdestroy,                /* eevo_qdestroy */
 118         falconsiena_ev_qprime,                  /* eevo_qprime */
 119         falconsiena_ev_qpost,                   /* eevo_qpost */
 120         falconsiena_ev_qmoderate,               /* eevo_qmoderate */
 121 #if EFSYS_OPT_QSTATS
 122         falconsiena_ev_qstats_update,           /* eevo_qstats_update */
 123 #endif
 124 };
 125 #endif /* EFSYS_OPT_FALCON */
 126
 127 #if EFSYS_OPT_SIENA
 128 static efx_ev_ops_t     __efx_ev_siena_ops = {
 129         falconsiena_ev_init,                    /* eevo_init */
 130         falconsiena_ev_fini,                    /* eevo_fini */
 131         falconsiena_ev_qcreate,                 /* eevo_qcreate */
 132         falconsiena_ev_qdestroy,                /* eevo_qdestroy */
 133         falconsiena_ev_qprime,                  /* eevo_qprime */
 134         falconsiena_ev_qpost,                   /* eevo_qpost */
 135         falconsiena_ev_qmoderate,               /* eevo_qmoderate */
 136 #if EFSYS_OPT_QSTATS
 137         falconsiena_ev_qstats_update,           /* eevo_qstats_update */
 138 #endif
 139 };
 140 #endif /* EFSYS_OPT_SIENA */
 141
 142 #if EFSYS_OPT_HUNTINGTON
 143 static efx_ev_ops_t     __efx_ev_hunt_ops = {
 144         hunt_ev_init,                           /* eevo_init */
 145         hunt_ev_fini,                           /* eevo_fini */
 146         hunt_ev_qcreate,                        /* eevo_qcreate */
 147         hunt_ev_qdestroy,                       /* eevo_qdestroy */
 148         hunt_ev_qprime,                         /* eevo_qprime */
 149         hunt_ev_qpost,                          /* eevo_qpost */
 150         hunt_ev_qmoderate,                      /* eevo_qmoderate */
 151 #if EFSYS_OPT_QSTATS
 152         hunt_ev_qstats_update,                  /* eevo_qstats_update */
 153 #endif
 154 };
 155 #endif /* EFSYS_OPT_HUNTINGTON */
 156
 157
 158         __checkReturn   int
 159 efx_ev_init(
 160         __in            efx_nic_t *enp)
 161 {
 162         efx_ev_ops_t *eevop;
 163         int rc;
 164
 165         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 166         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 167
 168         if (enp->en_mod_flags & EFX_MOD_EV) {
 169                 rc = EINVAL;
 170                 goto fail1;
 171         }
 172
 173         switch (enp->en_family) {
 174 #if EFSYS_OPT_FALCON
 175         case EFX_FAMILY_FALCON:
 176                 eevop = (efx_ev_ops_t *)&__efx_ev_falcon_ops;
 177                 break;
 178 #endif /* EFSYS_OPT_FALCON */
 179
 180 #if EFSYS_OPT_SIENA
 181         case EFX_FAMILY_SIENA:
 182                 eevop = (efx_ev_ops_t *)&__efx_ev_siena_ops;
 183                 break;
 184 #endif /* EFSYS_OPT_SIENA */
 185
 186 #if EFSYS_OPT_HUNTINGTON
 187         case EFX_FAMILY_HUNTINGTON:
 188                 eevop = (efx_ev_ops_t *)&__efx_ev_hunt_ops;
 189                 break;
 190 #endif /* EFSYS_OPT_HUNTINGTON */
 191
 192         default:
 193                 EFSYS_ASSERT(0);
 194                 rc = ENOTSUP;
 195                 goto fail1;
 196         }
 197
 198         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 199
 200         if ((rc = eevop->eevo_init(enp)) != 0)
 201                 goto fail2;
 202
 203         enp->en_eevop = eevop;
 204         enp->en_mod_flags |= EFX_MOD_EV;
 205         return (0);
 206
 207 fail2:
 208         EFSYS_PROBE(fail2);
 209
 210 fail1:
 211         EFSYS_PROBE1(fail1, int, rc);
 212
 213         enp->en_eevop = NULL;
 214         enp->en_mod_flags &= ~EFX_MOD_EV;
 215         return (rc);
 216 }
 217
 218                 void
 219 efx_ev_fini(
 220         __in    efx_nic_t *enp)
 221 {
 222         efx_ev_ops_t *eevop = enp->en_eevop;
 223
 224         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 225         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 226         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 227         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
 228         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
 229         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 230
 231         eevop->eevo_fini(enp);
 232
 233         enp->en_eevop = NULL;
 234         enp->en_mod_flags &= ~EFX_MOD_EV;
 235 }
 236
 237
 238         __checkReturn   int
 239 efx_ev_qcreate(
 240         __in            efx_nic_t *enp,
 241         __in            unsigned int index,
 242         __in            efsys_mem_t *esmp,
 243         __in            size_t n,
 244         __in            uint32_t id,
 245         __deref_out     efx_evq_t **eepp)
 246 {
 247         efx_ev_ops_t *eevop = enp->en_eevop;
 248         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 249         efx_evq_t *eep;
 250         int rc;
 251
 252         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 253         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 254
 255         EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
 256
 257         /* Allocate an EVQ object */
 258         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
 259         if (eep == NULL) {
 260                 rc = ENOMEM;
 261                 goto fail1;
 262         }
 263
 264         eep->ee_magic = EFX_EVQ_MAGIC;
 265         eep->ee_enp = enp;
 266         eep->ee_index = index;
 267         eep->ee_mask = n - 1;
 268         eep->ee_esmp = esmp;
 269
 270         if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, eep)) != 0)
 271                 goto fail2;
 272
 273         enp->en_ev_qcount++;
 274         *eepp = eep;
 275
 276         return (0);
 277
 278 fail2:
 279         EFSYS_PROBE(fail2);
 280         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 281 fail1:
 282         EFSYS_PROBE1(fail1, int, rc);
 283         return (rc);
 284 }
 285
 286                 void
 287 efx_ev_qdestroy(
 288         __in    efx_evq_t *eep)
 289 {
 290         efx_nic_t *enp = eep->ee_enp;
 291         efx_ev_ops_t *eevop = enp->en_eevop;
 292
 293         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 294
 295         EFSYS_ASSERT(enp->en_ev_qcount != 0);
 296         --enp->en_ev_qcount;
 297
 298         eevop->eevo_qdestroy(eep);
 299
 300         /* Free the EVQ object */
 301         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 302 }
 303
 304         __checkReturn   int
 305 efx_ev_qprime(
 306         __in            efx_evq_t *eep,
 307         __in            unsigned int count)
 308 {
 309         efx_nic_t *enp = eep->ee_enp;
 310         efx_ev_ops_t *eevop = enp->en_eevop;
 311         int rc;
 312
 313         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 314
 315         if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
 316                 rc = EINVAL;
 317                 goto fail1;
 318         }
 319
 320         if ((rc = eevop->eevo_qprime(eep, count)) != 0)
 321                 goto fail2;
 322
 323         return (0);
 324
 325 fail2:
 326         EFSYS_PROBE(fail2);
 327 fail1:
 328         EFSYS_PROBE1(fail1, int, rc);
 329         return (rc);
 330 }
 331
 332         __checkReturn   boolean_t
 333 efx_ev_qpending(
 334         __in            efx_evq_t *eep,
 335         __in            unsigned int count)
 336 {
 337         size_t offset;
 338         efx_qword_t qword;
 339
 340         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 341
 342         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 343         EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
 344
 345         return (EFX_EV_PRESENT(qword));
 346 }
 347
 348 #if EFSYS_OPT_EV_PREFETCH
 349
 350                         void
 351 efx_ev_qprefetch(
 352         __in            efx_evq_t *eep,
 353         __in            unsigned int count)
 354 {
 355         efx_nic_t *enp = eep->ee_enp;
 356         unsigned int offset;
 357
 358         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 359
 360         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 361         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 362 }
 363
 364 #endif  /* EFSYS_OPT_EV_PREFETCH */
 365
 366                         void
 367 efx_ev_qpoll(
 368         __in            efx_evq_t *eep,
 369         __inout         unsigned int *countp,
 370         __in            const efx_ev_callbacks_t *eecp,
 371         __in_opt        void *arg)
 372 {
 373         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 374
 375         /*
 376          * FIXME: Huntington will require support for hardware event batching
 377          * and merging, which will need a different ev_qpoll implementation.
 378          *
 379          * Without those features the Falcon/Siena code can be used unchanged.
 380          */
 381         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
 382         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
 383
 384         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
 385         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
 386         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
 387         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
 388             FSE_AZ_EV_CODE_DRV_GEN_EV);
 389 #if EFSYS_OPT_MCDI
 390         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
 391             FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
 392 #endif
 393         falconsiena_ev_qpoll(eep, countp, eecp, arg);
 394 }
 395
 396                         void
 397 efx_ev_qpost(
 398         __in    efx_evq_t *eep,
 399         __in    uint16_t data)
 400 {
 401         efx_nic_t *enp = eep->ee_enp;
 402         efx_ev_ops_t *eevop = enp->en_eevop;
 403
 404         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 405
 406         EFSYS_ASSERT(eevop != NULL &&
 407             eevop->eevo_qpost != NULL);
 408
 409         eevop->eevo_qpost(eep, data);
 410 }
 411
 412         __checkReturn   int
 413 efx_ev_qmoderate(
 414         __in            efx_evq_t *eep,
 415         __in            unsigned int us)
 416 {
 417         efx_nic_t *enp = eep->ee_enp;
 418         efx_ev_ops_t *eevop = enp->en_eevop;
 419         int rc;
 420
 421         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 422
 423         if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
 424                 goto fail1;
 425
 426         return (0);
 427
 428 fail1:
 429         EFSYS_PROBE1(fail1, int, rc);
 430         return (rc);
 431 }
 432
 433 #if EFSYS_OPT_QSTATS
 434                                         void
 435 efx_ev_qstats_update(
 436         __in                            efx_evq_t *eep,
 437         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
 438
 439 {       efx_nic_t *enp = eep->ee_enp;
 440         efx_ev_ops_t *eevop = enp->en_eevop;
 441
 442         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 443
 444         eevop->eevo_qstats_update(eep, stat);
 445 }
 446
 447 #endif  /* EFSYS_OPT_QSTATS */
 448
 449 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
 450
 451 static  __checkReturn   int
 452 falconsiena_ev_init(
 453         __in            efx_nic_t *enp)
 454 {
 455         efx_oword_t oword;
 456
 457         /*
 458          * Program the event queue for receive and transmit queue
 459          * flush events.
 460          */
 461         EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
 462         EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
 463         EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
 464
 465         return (0);
 466
 467 }
 468
 469 static  __checkReturn   boolean_t
 470 falconsiena_ev_rx_not_ok(
 471         __in            efx_evq_t *eep,
 472         __in            efx_qword_t *eqp,
 473         __in            uint32_t label,
 474         __in            uint32_t id,
 475         __inout         uint16_t *flagsp)
 476 {
 477         boolean_t ignore = B_FALSE;
 478
 479         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
 480                 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
 481                 EFSYS_PROBE(tobe_disc);
 482                 /*
 483                  * Assume this is a unicast address mismatch, unless below
 484                  * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
 485                  * EV_RX_PAUSE_FRM_ERR is set.
 486                  */
 487                 (*flagsp) |= EFX_ADDR_MISMATCH;
 488         }
 489
 490         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
 491                 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
 492                 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
 493                 (*flagsp) |= EFX_DISCARD;
 494
 495 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 496                 /*
 497                  * Lookout for payload queue ran dry errors and ignore them.
 498                  *
 499                  * Sadly for the header/data split cases, the descriptor
 500                  * pointer in this event refers to the header queue and
 501                  * therefore cannot be easily detected as duplicate.
 502                  * So we drop these and rely on the receive processing seeing
 503                  * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
 504                  * the partially received packet.
 505                  */
 506                 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
 507                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
 508                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
 509                         ignore = B_TRUE;
 510 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 511         }
 512
 513         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
 514                 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
 515                 EFSYS_PROBE(crc_err);
 516                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 517                 (*flagsp) |= EFX_DISCARD;
 518         }
 519
 520         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
 521                 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
 522                 EFSYS_PROBE(pause_frm_err);
 523                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 524                 (*flagsp) |= EFX_DISCARD;
 525         }
 526
 527         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
 528                 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
 529                 EFSYS_PROBE(owner_id_err);
 530                 (*flagsp) |= EFX_DISCARD;
 531         }
 532
 533         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
 534                 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
 535                 EFSYS_PROBE(ipv4_err);
 536                 (*flagsp) &= ~EFX_CKSUM_IPV4;
 537         }
 538
 539         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
 540                 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
 541                 EFSYS_PROBE(udp_chk_err);
 542                 (*flagsp) &= ~EFX_CKSUM_TCPUDP;
 543         }
 544
 545         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
 546                 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
 547
 548                 /*
 549                  * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
 550                  * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
 551                  * condition.
 552                  */
 553                 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
 554         }
 555
 556         return (ignore);
 557 }
 558
 559 static  __checkReturn   boolean_t
 560 falconsiena_ev_rx(
 561         __in            efx_evq_t *eep,
 562         __in            efx_qword_t *eqp,
 563         __in            const efx_ev_callbacks_t *eecp,
 564         __in_opt        void *arg)
 565 {
 566         efx_nic_t *enp = eep->ee_enp;
 567         uint32_t id;
 568         uint32_t size;
 569         uint32_t label;
 570         boolean_t ok;
 571 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 572         boolean_t sop;
 573         boolean_t jumbo_cont;
 574 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 575         uint32_t hdr_type;
 576         boolean_t is_v6;
 577         uint16_t flags;
 578         boolean_t ignore;
 579         boolean_t should_abort;
 580
 581         EFX_EV_QSTAT_INCR(eep, EV_RX);
 582
 583         /* Basic packet information */
 584         id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
 585         size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
 586         label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
 587         ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
 588
 589 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 590         sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
 591         jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
 592 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 593
 594         hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
 595
 596         is_v6 = (enp->en_family != EFX_FAMILY_FALCON &&
 597                     EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
 598
 599         /*
 600          * If packet is marked as OK and packet type is TCP/IP or
 601          * UDP/IP or other IP, then we can rely on the hardware checksums.
 602          */
 603         switch (hdr_type) {
 604         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
 605                 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
 606                 if (is_v6) {
 607                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
 608                         flags |= EFX_PKT_IPV6;
 609                 } else {
 610                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
 611                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 612                 }
 613                 break;
 614
 615         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
 616                 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
 617                 if (is_v6) {
 618                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
 619                         flags |= EFX_PKT_IPV6;
 620                 } else {
 621                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
 622                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 623                 }
 624                 break;
 625
 626         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
 627                 if (is_v6) {
 628                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
 629                         flags = EFX_PKT_IPV6;
 630                 } else {
 631                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
 632                         flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 633                 }
 634                 break;
 635
 636         case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
 637                 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
 638                 flags = 0;
 639                 break;
 640
 641         default:
 642                 EFSYS_ASSERT(B_FALSE);
 643                 flags = 0;
 644                 break;
 645         }
 646
 647 #if EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT
 648         /* Report scatter and header/lookahead split buffer flags */
 649         if (sop)
 650                 flags |= EFX_PKT_START;
 651         if (jumbo_cont)
 652                 flags |= EFX_PKT_CONT;
 653 #endif  /* EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT */
 654
 655         /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
 656         if (!ok) {
 657                 ignore = falconsiena_ev_rx_not_ok(eep, eqp, label, id, &flags);
 658                 if (ignore) {
 659                         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 660                             uint32_t, size, uint16_t, flags);
 661
 662                         return (B_FALSE);
 663                 }
 664         }
 665
 666         /* If we're not discarding the packet then it is ok */
 667         if (~flags & EFX_DISCARD)
 668                 EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
 669
 670         /* Detect multicast packets that didn't match the filter */
 671         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
 672                 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
 673
 674                 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
 675                         EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
 676                 } else {
 677                         EFSYS_PROBE(mcast_mismatch);
 678                         flags |= EFX_ADDR_MISMATCH;
 679                 }
 680         } else {
 681                 flags |= EFX_PKT_UNICAST;
 682         }
 683
 684         /*
 685          * The packet parser in Siena can abort parsing packets under
 686          * certain error conditions, setting the PKT_NOT_PARSED bit
 687          * (which clears PKT_OK). If this is set, then don't trust
 688          * the PKT_TYPE field.
 689          */
 690         if (enp->en_family != EFX_FAMILY_FALCON && !ok) {
 691                 uint32_t parse_err;
 692
 693                 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
 694                 if (parse_err != 0)
 695                         flags |= EFX_CHECK_VLAN;
 696         }
 697
 698         if (~flags & EFX_CHECK_VLAN) {
 699                 uint32_t pkt_type;
 700
 701                 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
 702                 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
 703                         flags |= EFX_PKT_VLAN_TAGGED;
 704         }
 705
 706         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 707             uint32_t, size, uint16_t, flags);
 708
 709         EFSYS_ASSERT(eecp->eec_rx != NULL);
 710         should_abort = eecp->eec_rx(arg, label, id, size, flags);
 711
 712         return (should_abort);
 713 }
 714
 715 static  __checkReturn   boolean_t
 716 falconsiena_ev_tx(
 717         __in            efx_evq_t *eep,
 718         __in            efx_qword_t *eqp,
 719         __in            const efx_ev_callbacks_t *eecp,
 720         __in_opt        void *arg)
 721 {
 722         uint32_t id;
 723         uint32_t label;
 724         boolean_t should_abort;
 725
 726         EFX_EV_QSTAT_INCR(eep, EV_TX);
 727
 728         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
 729             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
 730             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
 731             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
 732
 733                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
 734                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
 735
 736                 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
 737
 738                 EFSYS_ASSERT(eecp->eec_tx != NULL);
 739                 should_abort = eecp->eec_tx(arg, label, id);
 740
 741                 return (should_abort);
 742         }
 743
 744         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
 745                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 746                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 747                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 748
 749         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
 750                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
 751
 752         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
 753                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
 754
 755         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
 756                 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
 757
 758         EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
 759         return (B_FALSE);
 760 }
 761
 762 static  __checkReturn   boolean_t
 763 falconsiena_ev_global(
 764         __in            efx_evq_t *eep,
 765         __in            efx_qword_t *eqp,
 766         __in            const efx_ev_callbacks_t *eecp,
 767         __in_opt        void *arg)
 768 {
 769         efx_nic_t *enp = eep->ee_enp;
 770         efx_port_t *epp = &(enp->en_port);
 771         boolean_t should_abort;
 772
 773         EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
 774         should_abort = B_FALSE;
 775
 776         /* Check for a link management event */
 777         if (EFX_QWORD_FIELD(*eqp, FSF_BZ_GLB_EV_XG_MNT_INTR) != 0) {
 778                 EFX_EV_QSTAT_INCR(eep, EV_GLOBAL_MNT);
 779
 780                 EFSYS_PROBE(xg_mgt);
 781
 782                 epp->ep_mac_poll_needed = B_TRUE;
 783         }
 784
 785         return (should_abort);
 786 }
 787
 788 static  __checkReturn   boolean_t
 789 falconsiena_ev_driver(
 790         __in            efx_evq_t *eep,
 791         __in            efx_qword_t *eqp,
 792         __in            const efx_ev_callbacks_t *eecp,
 793         __in_opt        void *arg)
 794 {
 795         boolean_t should_abort;
 796
 797         EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
 798         should_abort = B_FALSE;
 799
 800         switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
 801         case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
 802                 uint32_t txq_index;
 803
 804                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
 805
 806                 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 807
 808                 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
 809
 810                 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
 811                 should_abort = eecp->eec_txq_flush_done(arg, txq_index);
 812
 813                 break;
 814         }
 815         case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
 816                 uint32_t rxq_index;
 817                 uint32_t failed;
 818
 819                 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
 820                 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
 821
 822                 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
 823                 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
 824
 825                 if (failed) {
 826                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
 827
 828                         EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
 829
 830                         should_abort = eecp->eec_rxq_flush_failed(arg,
 831                                                                     rxq_index);
 832                 } else {
 833                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
 834
 835                         EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
 836
 837                         should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
 838                 }
 839
 840                 break;
 841         }
 842         case FSE_AZ_EVQ_INIT_DONE_EV:
 843                 EFSYS_ASSERT(eecp->eec_initialized != NULL);
 844                 should_abort = eecp->eec_initialized(arg);
 845
 846                 break;
 847
 848         case FSE_AZ_EVQ_NOT_EN_EV:
 849                 EFSYS_PROBE(evq_not_en);
 850                 break;
 851
 852         case FSE_AZ_SRM_UPD_DONE_EV: {
 853                 uint32_t code;
 854
 855                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
 856
 857                 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 858
 859                 EFSYS_ASSERT(eecp->eec_sram != NULL);
 860                 should_abort = eecp->eec_sram(arg, code);
 861
 862                 break;
 863         }
 864         case FSE_AZ_WAKE_UP_EV: {
 865                 uint32_t id;
 866
 867                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 868
 869                 EFSYS_ASSERT(eecp->eec_wake_up != NULL);
 870                 should_abort = eecp->eec_wake_up(arg, id);
 871
 872                 break;
 873         }
 874         case FSE_AZ_TX_PKT_NON_TCP_UDP:
 875                 EFSYS_PROBE(tx_pkt_non_tcp_udp);
 876                 break;
 877
 878         case FSE_AZ_TIMER_EV: {
 879                 uint32_t id;
 880
 881                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 882
 883                 EFSYS_ASSERT(eecp->eec_timer != NULL);
 884                 should_abort = eecp->eec_timer(arg, id);
 885
 886                 break;
 887         }
 888         case FSE_AZ_RX_DSC_ERROR_EV:
 889                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
 890
 891                 EFSYS_PROBE(rx_dsc_error);
 892
 893                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 894                 should_abort = eecp->eec_exception(arg,
 895                         EFX_EXCEPTION_RX_DSC_ERROR, 0);
 896
 897                 break;
 898
 899         case FSE_AZ_TX_DSC_ERROR_EV:
 900                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
 901
 902                 EFSYS_PROBE(tx_dsc_error);
 903
 904                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 905                 should_abort = eecp->eec_exception(arg,
 906                         EFX_EXCEPTION_TX_DSC_ERROR, 0);
 907
 908                 break;
 909
 910         default:
 911                 break;
 912         }
 913
 914         return (should_abort);
 915 }
 916
 917 static  __checkReturn   boolean_t
 918 falconsiena_ev_drv_gen(
 919         __in            efx_evq_t *eep,
 920         __in            efx_qword_t *eqp,
 921         __in            const efx_ev_callbacks_t *eecp,
 922         __in_opt        void *arg)
 923 {
 924         uint32_t data;
 925         boolean_t should_abort;
 926
 927         EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
 928
 929         data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
 930         if (data >= ((uint32_t)1 << 16)) {
 931                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 932                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 933                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 934                 return (B_TRUE);
 935         }
 936
 937         EFSYS_ASSERT(eecp->eec_software != NULL);
 938         should_abort = eecp->eec_software(arg, (uint16_t)data);
 939
 940         return (should_abort);
 941 }
 942
 943 #if EFSYS_OPT_MCDI
 944
 945 static  __checkReturn   boolean_t
 946 falconsiena_ev_mcdi(
 947         __in            efx_evq_t *eep,
 948         __in            efx_qword_t *eqp,
 949         __in            const efx_ev_callbacks_t *eecp,
 950         __in_opt        void *arg)
 951 {
 952         efx_nic_t *enp = eep->ee_enp;
 953         unsigned code;
 954         boolean_t should_abort = B_FALSE;
 955
 956         EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
 957
 958         if (enp->en_family != EFX_FAMILY_SIENA)
 959                 goto out;
 960
 961         EFSYS_ASSERT(eecp->eec_link_change != NULL);
 962         EFSYS_ASSERT(eecp->eec_exception != NULL);
 963 #if EFSYS_OPT_MON_STATS
 964         EFSYS_ASSERT(eecp->eec_monitor != NULL);
 965 #endif
 966
 967         EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
 968
 969         code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
 970         switch (code) {
 971         case MCDI_EVENT_CODE_BADSSERT:
 972                 efx_mcdi_ev_death(enp, EINTR);
 973                 break;
 974
 975         case MCDI_EVENT_CODE_CMDDONE:
 976                 efx_mcdi_ev_cpl(enp,
 977                     MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
 978                     MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
 979                     MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
 980                 break;
 981
 982         case MCDI_EVENT_CODE_LINKCHANGE: {
 983                 efx_link_mode_t link_mode;
 984
 985                 siena_phy_link_ev(enp, eqp, &link_mode);
 986                 should_abort = eecp->eec_link_change(arg, link_mode);
 987                 break;
 988         }
 989         case MCDI_EVENT_CODE_SENSOREVT: {
 990 #if EFSYS_OPT_MON_STATS
 991                 efx_mon_stat_t id;
 992                 efx_mon_stat_value_t value;
 993                 int rc;
 994
 995                 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
 996                         should_abort = eecp->eec_monitor(arg, id, value);
 997                 else if (rc == ENOTSUP) {
 998                         should_abort = eecp->eec_exception(arg,
 999                                 EFX_EXCEPTION_UNKNOWN_SENSOREVT,
1000                                 MCDI_EV_FIELD(eqp, DATA));
1001                 } else
1002                         EFSYS_ASSERT(rc == ENODEV);     /* Wrong port */
1003 #else
1004                 should_abort = B_FALSE;
1005 #endif
1006                 break;
1007         }
1008         case MCDI_EVENT_CODE_SCHEDERR:
1009                 /* Informational only */
1010                 break;
1011
1012         case MCDI_EVENT_CODE_REBOOT:
1013                 efx_mcdi_ev_death(enp, EIO);
1014                 break;
1015
1016         case MCDI_EVENT_CODE_MAC_STATS_DMA:
1017 #if EFSYS_OPT_MAC_STATS
1018                 if (eecp->eec_mac_stats != NULL) {
1019                         eecp->eec_mac_stats(arg,
1020                             MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
1021                 }
1022 #endif
1023                 break;
1024
1025         case MCDI_EVENT_CODE_FWALERT: {
1026                 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
1027
1028                 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
1029                         should_abort = eecp->eec_exception(arg,
1030                                 EFX_EXCEPTION_FWALERT_SRAM,
1031                                 MCDI_EV_FIELD(eqp, FWALERT_DATA));
1032                 else
1033                         should_abort = eecp->eec_exception(arg,
1034                                 EFX_EXCEPTION_UNKNOWN_FWALERT,
1035                                 MCDI_EV_FIELD(eqp, DATA));
1036                 break;
1037         }
1038
1039         default:
1040                 EFSYS_PROBE1(mc_pcol_error, int, code);
1041                 break;
1042         }
1043
1044 out:
1045         return (should_abort);
1046 }
1047
1048 #endif  /* EFSYS_OPT_MCDI */
1049
1050 static  __checkReturn   int
1051 falconsiena_ev_qprime(
1052         __in            efx_evq_t *eep,
1053         __in            unsigned int count)
1054 {
1055         efx_nic_t *enp = eep->ee_enp;
1056         uint32_t rptr;
1057         efx_dword_t dword;
1058
1059         rptr = count & eep->ee_mask;
1060
1061         EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
1062
1063         EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
1064                             &dword, B_FALSE);
1065
1066         return (0);
1067 }
1068
1069 #define EFX_EV_BATCH    8
1070
1071 static                  void
1072 falconsiena_ev_qpoll(
1073         __in            efx_evq_t *eep,
1074         __inout         unsigned int *countp,
1075         __in            const efx_ev_callbacks_t *eecp,
1076         __in_opt        void *arg)
1077 {
1078         efx_qword_t ev[EFX_EV_BATCH];
1079         unsigned int batch;
1080         unsigned int total;
1081         unsigned int count;
1082         unsigned int index;
1083         size_t offset;
1084
1085         EFSYS_ASSERT(countp != NULL);
1086         EFSYS_ASSERT(eecp != NULL);
1087
1088         count = *countp;
1089         do {
1090                 /* Read up until the end of the batch period */
1091                 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
1092                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1093                 for (total = 0; total < batch; ++total) {
1094                         EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
1095
1096                         if (!EFX_EV_PRESENT(ev[total]))
1097                                 break;
1098
1099                         EFSYS_PROBE3(event, unsigned int, eep->ee_index,
1100                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
1101                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
1102
1103                         offset += sizeof (efx_qword_t);
1104                 }
1105
1106 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
1107                 /*
1108                  * Prefetch the next batch when we get within PREFETCH_PERIOD
1109                  * of a completed batch. If the batch is smaller, then prefetch
1110                  * immediately.
1111                  */
1112                 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
1113                         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1114 #endif  /* EFSYS_OPT_EV_PREFETCH */
1115
1116                 /* Process the batch of events */
1117                 for (index = 0; index < total; ++index) {
1118                         boolean_t should_abort;
1119                         uint32_t code;
1120
1121 #if EFSYS_OPT_EV_PREFETCH
1122                         /* Prefetch if we've now reached the batch period */
1123                         if (total == batch &&
1124                             index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
1125                                 offset = (count + batch) & eep->ee_mask;
1126                                 offset *= sizeof (efx_qword_t);
1127
1128                                 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1129                         }
1130 #endif  /* EFSYS_OPT_EV_PREFETCH */
1131
1132                         EFX_EV_QSTAT_INCR(eep, EV_ALL);
1133
1134                         code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
1135                         switch (code) {
1136                         case FSE_AZ_EV_CODE_RX_EV:
1137                                 should_abort = eep->ee_rx(eep,
1138                                     &(ev[index]), eecp, arg);
1139                                 break;
1140                         case FSE_AZ_EV_CODE_TX_EV:
1141                                 should_abort = eep->ee_tx(eep,
1142                                     &(ev[index]), eecp, arg);
1143                                 break;
1144                         case FSE_AZ_EV_CODE_DRIVER_EV:
1145                                 should_abort = eep->ee_driver(eep,
1146                                     &(ev[index]), eecp, arg);
1147                                 break;
1148                         case FSE_AZ_EV_CODE_DRV_GEN_EV:
1149                                 should_abort = eep->ee_drv_gen(eep,
1150                                     &(ev[index]), eecp, arg);
1151                                 break;
1152 #if EFSYS_OPT_MCDI
1153                         case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
1154                                 should_abort = eep->ee_mcdi(eep,
1155                                     &(ev[index]), eecp, arg);
1156                                 break;
1157 #endif
1158                         case FSE_AZ_EV_CODE_GLOBAL_EV:
1159                                 if (eep->ee_global) {
1160                                         should_abort = eep->ee_global(eep,
1161                                             &(ev[index]), eecp, arg);
1162                                         break;
1163                                 }
1164                                 /* else fallthrough */
1165                         default:
1166                                 EFSYS_PROBE3(bad_event,
1167                                     unsigned int, eep->ee_index,
1168                                     uint32_t,
1169                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
1170                                     uint32_t,
1171                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
1172
1173                                 EFSYS_ASSERT(eecp->eec_exception != NULL);
1174                                 (void) eecp->eec_exception(arg,
1175                                         EFX_EXCEPTION_EV_ERROR, code);
1176                                 should_abort = B_TRUE;
1177                         }
1178                         if (should_abort) {
1179                                 /* Ignore subsequent events */
1180                                 total = index + 1;
1181                                 break;
1182                         }
1183                 }
1184
1185                 /*
1186                  * Now that the hardware has most likely moved onto dma'ing
1187                  * into the next cache line, clear the processed events. Take
1188                  * care to only clear out events that we've processed
1189                  */
1190                 EFX_SET_QWORD(ev[0]);
1191                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1192                 for (index = 0; index < total; ++index) {
1193                         EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
1194                         offset += sizeof (efx_qword_t);
1195                 }
1196
1197                 count += total;
1198
1199         } while (total == batch);
1200
1201         *countp = count;
1202 }
1203
1204 static          void
1205 falconsiena_ev_qpost(
1206         __in    efx_evq_t *eep,
1207         __in    uint16_t data)
1208 {
1209         efx_nic_t *enp = eep->ee_enp;
1210         efx_qword_t ev;
1211         efx_oword_t oword;
1212
1213         EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
1214             FSF_AZ_EV_DATA_DW0, (uint32_t)data);
1215
1216         EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
1217             EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
1218             EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
1219
1220         EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
1221 }
1222
1223 static  __checkReturn   int
1224 falconsiena_ev_qmoderate(
1225         __in            efx_evq_t *eep,
1226         __in            unsigned int us)
1227 {
1228         efx_nic_t *enp = eep->ee_enp;
1229         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1230         unsigned int locked;
1231         efx_dword_t dword;
1232         int rc;
1233
1234         if (us > encp->enc_evq_timer_max_us) {
1235                 rc = EINVAL;
1236                 goto fail1;
1237         }
1238
1239         /* If the value is zero then disable the timer */
1240         if (us == 0) {
1241                 if (enp->en_family == EFX_FAMILY_FALCON)
1242                         EFX_POPULATE_DWORD_2(dword,
1243                             FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_DIS,
1244                             FRF_AB_TC_TIMER_VAL, 0);
1245                 else
1246                         EFX_POPULATE_DWORD_2(dword,
1247                             FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
1248                             FRF_CZ_TC_TIMER_VAL, 0);
1249         } else {
1250                 uint32_t timer_val;
1251
1252                 /* Calculate the timer value in quanta */
1253                 timer_val = us * 1000 / encp->enc_evq_timer_quantum_ns;
1254
1255                 /* Moderation value is base 0 so we need to deduct 1 */
1256                 if (timer_val > 0)
1257                         timer_val--;
1258
1259                 if (enp->en_family == EFX_FAMILY_FALCON)
1260                         EFX_POPULATE_DWORD_2(dword,
1261                             FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_INT_HLDOFF,
1262                             FRF_AB_TIMER_VAL, timer_val);
1263                 else
1264                         EFX_POPULATE_DWORD_2(dword,
1265                             FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
1266                             FRF_CZ_TC_TIMER_VAL, timer_val);
1267         }
1268
1269         locked = (eep->ee_index == 0) ? 1 : 0;
1270
1271         EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
1272             eep->ee_index, &dword, locked);
1273
1274         return (0);
1275
1276 fail1:
1277         EFSYS_PROBE1(fail1, int, rc);
1278
1279         return (rc);
1280 }
1281
1282 static  __checkReturn   int
1283 falconsiena_ev_qcreate(
1284         __in            efx_nic_t *enp,
1285         __in            unsigned int index,
1286         __in            efsys_mem_t *esmp,
1287         __in            size_t n,
1288         __in            uint32_t id,
1289         __in            efx_evq_t *eep)
1290 {
1291         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1292         uint32_t size;
1293         efx_oword_t oword;
1294         int rc;
1295
1296         EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS));
1297         EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS));
1298
1299         if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) {
1300                 rc = EINVAL;
1301                 goto fail1;
1302         }
1303         if (index >= encp->enc_evq_limit) {
1304                 rc = EINVAL;
1305                 goto fail2;
1306         }
1307 #if EFSYS_OPT_RX_SCALE
1308         if (enp->en_intr.ei_type == EFX_INTR_LINE &&
1309             index >= EFX_MAXRSS_LEGACY) {
1310                 rc = EINVAL;
1311                 goto fail3;
1312         }
1313 #endif
1314         for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
1315             size++)
1316                 if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
1317                         break;
1318         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1319                 rc = EINVAL;
1320                 goto fail4;
1321         }
1322
1323         /* Set up the handler table */
1324         eep->ee_rx      = falconsiena_ev_rx;
1325         eep->ee_tx      = falconsiena_ev_tx;
1326         eep->ee_driver  = falconsiena_ev_driver;
1327         eep->ee_global  = falconsiena_ev_global;
1328         eep->ee_drv_gen = falconsiena_ev_drv_gen;
1329 #if EFSYS_OPT_MCDI
1330         eep->ee_mcdi    = falconsiena_ev_mcdi;
1331 #endif  /* EFSYS_OPT_MCDI */
1332
1333         /* Set up the new event queue */
1334         if (enp->en_family != EFX_FAMILY_FALCON) {
1335                 EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1);
1336                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
1337         }
1338
1339         EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
1340             FRF_AZ_EVQ_BUF_BASE_ID, id);
1341
1342         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
1343
1344         return (0);
1345
1346 fail4:
1347         EFSYS_PROBE(fail4);
1348 #if EFSYS_OPT_RX_SCALE
1349 fail3:
1350         EFSYS_PROBE(fail3);
1351 #endif
1352 fail2:
1353         EFSYS_PROBE(fail2);
1354 fail1:
1355         EFSYS_PROBE1(fail1, int, rc);
1356
1357         return (rc);
1358 }
1359
1360 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1361
1362 #if EFSYS_OPT_QSTATS
1363 #if EFSYS_OPT_NAMES
1364 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock b693ddf85aee1bfd */
1365 static const char       *__efx_ev_qstat_name[] = {
1366         "all",
1367         "rx",
1368         "rx_ok",
1369         "rx_frm_trunc",
1370         "rx_tobe_disc",
1371         "rx_pause_frm_err",
1372         "rx_buf_owner_id_err",
1373         "rx_ipv4_hdr_chksum_err",
1374         "rx_tcp_udp_chksum_err",
1375         "rx_eth_crc_err",
1376         "rx_ip_frag_err",
1377         "rx_mcast_pkt",
1378         "rx_mcast_hash_match",
1379         "rx_tcp_ipv4",
1380         "rx_tcp_ipv6",
1381         "rx_udp_ipv4",
1382         "rx_udp_ipv6",
1383         "rx_other_ipv4",
1384         "rx_other_ipv6",
1385         "rx_non_ip",
1386         "rx_batch",
1387         "tx",
1388         "tx_wq_ff_full",
1389         "tx_pkt_err",
1390         "tx_pkt_too_big",
1391         "tx_unexpected",
1392         "global",
1393         "global_mnt",
1394         "driver",
1395         "driver_srm_upd_done",
1396         "driver_tx_descq_fls_done",
1397         "driver_rx_descq_fls_done",
1398         "driver_rx_descq_fls_failed",
1399         "driver_rx_dsc_error",
1400         "driver_tx_dsc_error",
1401         "drv_gen",
1402         "mcdi_response",
1403 };
1404 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1405
1406                 const char *
1407 efx_ev_qstat_name(
1408         __in    efx_nic_t *enp,
1409         __in    unsigned int id)
1410 {
1411         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1412         EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1413
1414         return (__efx_ev_qstat_name[id]);
1415 }
1416 #endif  /* EFSYS_OPT_NAMES */
1417 #endif  /* EFSYS_OPT_QSTATS */
1418
1419 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
1420
1421 #if EFSYS_OPT_QSTATS
1422 static                                  void
1423 falconsiena_ev_qstats_update(
1424         __in                            efx_evq_t *eep,
1425         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
1426 {
1427         unsigned int id;
1428
1429         for (id = 0; id < EV_NQSTATS; id++) {
1430                 efsys_stat_t *essp = &stat[id];
1431
1432                 EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1433                 eep->ee_stat[id] = 0;
1434         }
1435 }
1436 #endif  /* EFSYS_OPT_QSTATS */
1437
1438 static          void
1439 falconsiena_ev_qdestroy(
1440         __in    efx_evq_t *eep)
1441 {
1442         efx_nic_t *enp = eep->ee_enp;
1443         efx_oword_t oword;
1444
1445         /* Purge event queue */
1446         EFX_ZERO_OWORD(oword);
1447
1448         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1449             eep->ee_index, &oword, B_TRUE);
1450
1451         if (enp->en_family != EFX_FAMILY_FALCON) {
1452                 EFX_ZERO_OWORD(oword);
1453                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL,
1454                     eep->ee_index, &oword, B_TRUE);
1455         }
1456 }
1457
1458 static          void
1459 falconsiena_ev_fini(
1460         __in    efx_nic_t *enp)
1461 {
1462         _NOTE(ARGUNUSED(enp))
1463 }
1464
1465 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */