sys/dev/sfxge/common/efx_ev.c

   1 /*-
   2  * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25
  26 #include <sys/cdefs.h>
  27 __FBSDID("$FreeBSD$");
  28
  29 #include "efsys.h"
  30 #include "efx.h"
  31 #include "efx_types.h"
  32 #include "efx_regs.h"
  33 #include "efx_impl.h"
  34
  35 #if EFSYS_OPT_QSTATS
  36 #define EFX_EV_QSTAT_INCR(_eep, _stat)                                  \
  37         do {                                                            \
  38                 (_eep)->ee_stat[_stat]++;                               \
  39         _NOTE(CONSTANTCONDITION)                                        \
  40         } while (B_FALSE)
  41 #else
  42 #define EFX_EV_QSTAT_INCR(_eep, _stat)
  43 #endif
  44
  45         __checkReturn   int
  46 efx_ev_init(
  47         __in            efx_nic_t *enp)
  48 {
  49         efx_oword_t oword;
  50         int rc;
  51
  52         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
  53         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
  54
  55         if (enp->en_mod_flags & EFX_MOD_EV) {
  56                 rc = EINVAL;
  57                 goto fail1;
  58         }
  59
  60         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
  61
  62         /*
  63          * Program the event queue for receive and transmit queue
  64          * flush events.
  65          */
  66         EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
  67         EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
  68         EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
  69
  70         enp->en_mod_flags |= EFX_MOD_EV;
  71         return (0);
  72
  73 fail1:
  74         EFSYS_PROBE1(fail1, int, rc);
  75
  76         return (rc);
  77 }
  78
  79 static  __checkReturn   boolean_t
  80 efx_ev_rx_not_ok(
  81         __in            efx_evq_t *eep,
  82         __in            efx_qword_t *eqp,
  83         __in            uint32_t label,
  84         __in            uint32_t id,
  85         __inout         uint16_t *flagsp)
  86 {
  87         boolean_t ignore = B_FALSE;
  88
  89         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
  90                 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
  91                 EFSYS_PROBE(tobe_disc);
  92                 /*
  93                  * Assume this is a unicast address mismatch, unless below
  94                  * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
  95                  * EV_RX_PAUSE_FRM_ERR is set.
  96                  */
  97                 (*flagsp) |= EFX_ADDR_MISMATCH;
  98         }
  99
 100         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
 101                 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
 102                 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
 103                 (*flagsp) |= EFX_DISCARD;
 104
 105 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 106                 /*
 107                  * Lookout for payload queue ran dry errors and ignore them.
 108                  *
 109                  * Sadly for the header/data split cases, the descriptor
 110                  * pointer in this event refers to the header queue and
 111                  * therefore cannot be easily detected as duplicate.
 112                  * So we drop these and rely on the receive processing seeing
 113                  * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
 114                  * the partially received packet.
 115                  */
 116                 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
 117                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
 118                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
 119                         ignore = B_TRUE;
 120 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 121         }
 122
 123         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
 124                 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
 125                 EFSYS_PROBE(crc_err);
 126                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 127                 (*flagsp) |= EFX_DISCARD;
 128         }
 129
 130         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
 131                 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
 132                 EFSYS_PROBE(pause_frm_err);
 133                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 134                 (*flagsp) |= EFX_DISCARD;
 135         }
 136
 137         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
 138                 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
 139                 EFSYS_PROBE(owner_id_err);
 140                 (*flagsp) |= EFX_DISCARD;
 141         }
 142
 143         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
 144                 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
 145                 EFSYS_PROBE(ipv4_err);
 146                 (*flagsp) &= ~EFX_CKSUM_IPV4;
 147         }
 148
 149         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
 150                 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
 151                 EFSYS_PROBE(udp_chk_err);
 152                 (*flagsp) &= ~EFX_CKSUM_TCPUDP;
 153         }
 154
 155         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
 156                 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
 157
 158                 /*
 159                  * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
 160                  * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
 161                  * condition.
 162                  */
 163                 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
 164         }
 165
 166         return (ignore);
 167 }
 168
 169 static  __checkReturn   boolean_t
 170 efx_ev_rx(
 171         __in            efx_evq_t *eep,
 172         __in            efx_qword_t *eqp,
 173         __in            const efx_ev_callbacks_t *eecp,
 174         __in_opt        void *arg)
 175 {
 176         efx_nic_t *enp = eep->ee_enp;
 177         uint32_t id;
 178         uint32_t size;
 179         uint32_t label;
 180         boolean_t ok;
 181 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 182         boolean_t sop;
 183         boolean_t jumbo_cont;
 184 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 185         uint32_t hdr_type;
 186         boolean_t is_v6;
 187         uint16_t flags;
 188         boolean_t ignore;
 189         boolean_t should_abort;
 190
 191         EFX_EV_QSTAT_INCR(eep, EV_RX);
 192
 193         /* Basic packet information */
 194         id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
 195         size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
 196         label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
 197         ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
 198
 199 #if (EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER)
 200         sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
 201         jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
 202 #endif  /* EFSYS_OPT_RX_HDR_SPLIT || EFSYS_OPT_RX_SCATTER */
 203
 204         hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
 205
 206         is_v6 = (enp->en_family != EFX_FAMILY_FALCON &&
 207                     EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
 208
 209         /*
 210          * If packet is marked as OK and packet type is TCP/IP or
 211          * UDP/IP or other IP, then we can rely on the hardware checksums.
 212          */
 213         switch (hdr_type) {
 214         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
 215                 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
 216                 if (is_v6) {
 217                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
 218                         flags |= EFX_PKT_IPV6;
 219                 } else {
 220                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
 221                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 222                 }
 223                 break;
 224
 225         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
 226                 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
 227                 if (is_v6) {
 228                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
 229                         flags |= EFX_PKT_IPV6;
 230                 } else {
 231                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
 232                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 233                 }
 234                 break;
 235
 236         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
 237                 if (is_v6) {
 238                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
 239                         flags = EFX_PKT_IPV6;
 240                 } else {
 241                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
 242                         flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 243                 }
 244                 break;
 245
 246         case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
 247                 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
 248                 flags = 0;
 249                 break;
 250
 251         default:
 252                 EFSYS_ASSERT(B_FALSE);
 253                 flags = 0;
 254                 break;
 255         }
 256
 257 #if EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT
 258         /* Report scatter and header/lookahead split buffer flags */
 259         if (sop)
 260                 flags |= EFX_PKT_START;
 261         if (jumbo_cont)
 262                 flags |= EFX_PKT_CONT;
 263 #endif  /* EFSYS_OPT_RX_SCATTER || EFSYS_OPT_RX_HDR_SPLIT */
 264
 265         /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
 266         if (!ok) {
 267                 ignore = efx_ev_rx_not_ok(eep, eqp, label, id, &flags);
 268                 if (ignore) {
 269                         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 270                             uint32_t, size, uint16_t, flags);
 271
 272                         return (B_FALSE);
 273                 }
 274         }
 275
 276         /* If we're not discarding the packet then it is ok */
 277         if (~flags & EFX_DISCARD)
 278                 EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
 279
 280         /* Detect multicast packets that didn't match the filter */
 281         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
 282                 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
 283
 284                 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
 285                         EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
 286                 } else {
 287                         EFSYS_PROBE(mcast_mismatch);
 288                         flags |= EFX_ADDR_MISMATCH;
 289                 }
 290         } else {
 291                 flags |= EFX_PKT_UNICAST;
 292         }
 293
 294         /*
 295          * The packet parser in Siena can abort parsing packets under
 296          * certain error conditions, setting the PKT_NOT_PARSED bit
 297          * (which clears PKT_OK). If this is set, then don't trust
 298          * the PKT_TYPE field.
 299          */
 300         if (enp->en_family != EFX_FAMILY_FALCON && !ok) {
 301                 uint32_t parse_err;
 302
 303                 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
 304                 if (parse_err != 0)
 305                         flags |= EFX_CHECK_VLAN;
 306         }
 307
 308         if (~flags & EFX_CHECK_VLAN) {
 309                 uint32_t pkt_type;
 310
 311                 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
 312                 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
 313                         flags |= EFX_PKT_VLAN_TAGGED;
 314         }
 315
 316         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 317             uint32_t, size, uint16_t, flags);
 318
 319         EFSYS_ASSERT(eecp->eec_rx != NULL);
 320         should_abort = eecp->eec_rx(arg, label, id, size, flags);
 321
 322         return (should_abort);
 323 }
 324
 325 static  __checkReturn   boolean_t
 326 efx_ev_tx(
 327         __in            efx_evq_t *eep,
 328         __in            efx_qword_t *eqp,
 329         __in            const efx_ev_callbacks_t *eecp,
 330         __in_opt        void *arg)
 331 {
 332         uint32_t id;
 333         uint32_t label;
 334         boolean_t should_abort;
 335
 336         EFX_EV_QSTAT_INCR(eep, EV_TX);
 337
 338         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
 339             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
 340             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
 341             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
 342
 343                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
 344                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
 345
 346                 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
 347
 348                 EFSYS_ASSERT(eecp->eec_tx != NULL);
 349                 should_abort = eecp->eec_tx(arg, label, id);
 350
 351                 return (should_abort);
 352         }
 353
 354         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
 355                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 356                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 357                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 358
 359         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
 360                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
 361
 362         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
 363                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
 364
 365         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
 366                 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
 367
 368         EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
 369         return (B_FALSE);
 370 }
 371
 372 static  __checkReturn   boolean_t
 373 efx_ev_global(
 374         __in            efx_evq_t *eep,
 375         __in            efx_qword_t *eqp,
 376         __in            const efx_ev_callbacks_t *eecp,
 377         __in_opt        void *arg)
 378 {
 379         efx_nic_t *enp = eep->ee_enp;
 380         efx_port_t *epp = &(enp->en_port);
 381         boolean_t should_abort;
 382
 383         EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
 384         should_abort = B_FALSE;
 385
 386         /* Check for a link management event */
 387         if (EFX_QWORD_FIELD(*eqp, FSF_BZ_GLB_EV_XG_MNT_INTR) != 0) {
 388                 EFX_EV_QSTAT_INCR(eep, EV_GLOBAL_MNT);
 389
 390                 EFSYS_PROBE(xg_mgt);
 391
 392                 epp->ep_mac_poll_needed = B_TRUE;
 393         }
 394
 395         return (should_abort);
 396 }
 397
 398 static  __checkReturn   boolean_t
 399 efx_ev_driver(
 400         __in            efx_evq_t *eep,
 401         __in            efx_qword_t *eqp,
 402         __in            const efx_ev_callbacks_t *eecp,
 403         __in_opt        void *arg)
 404 {
 405         boolean_t should_abort;
 406
 407         EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
 408         should_abort = B_FALSE;
 409
 410         switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
 411         case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
 412                 uint32_t txq_index;
 413
 414                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
 415
 416                 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 417
 418                 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
 419
 420                 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
 421                 should_abort = eecp->eec_txq_flush_done(arg, txq_index);
 422
 423                 break;
 424         }
 425         case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
 426                 uint32_t rxq_index;
 427                 uint32_t failed;
 428
 429                 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
 430                 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
 431
 432                 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
 433                 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
 434
 435                 if (failed) {
 436                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
 437
 438                         EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
 439
 440                         should_abort = eecp->eec_rxq_flush_failed(arg, rxq_index);
 441                 } else {
 442                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
 443
 444                         EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
 445
 446                         should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
 447                 }
 448
 449                 break;
 450         }
 451         case FSE_AZ_EVQ_INIT_DONE_EV:
 452                 EFSYS_ASSERT(eecp->eec_initialized != NULL);
 453                 should_abort = eecp->eec_initialized(arg);
 454
 455                 break;
 456
 457         case FSE_AZ_EVQ_NOT_EN_EV:
 458                 EFSYS_PROBE(evq_not_en);
 459                 break;
 460
 461         case FSE_AZ_SRM_UPD_DONE_EV: {
 462                 uint32_t code;
 463
 464                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
 465
 466                 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 467
 468                 EFSYS_ASSERT(eecp->eec_sram != NULL);
 469                 should_abort = eecp->eec_sram(arg, code);
 470
 471                 break;
 472         }
 473         case FSE_AZ_WAKE_UP_EV: {
 474                 uint32_t id;
 475
 476                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 477
 478                 EFSYS_ASSERT(eecp->eec_wake_up != NULL);
 479                 should_abort = eecp->eec_wake_up(arg, id);
 480
 481                 break;
 482         }
 483         case FSE_AZ_TX_PKT_NON_TCP_UDP:
 484                 EFSYS_PROBE(tx_pkt_non_tcp_udp);
 485                 break;
 486
 487         case FSE_AZ_TIMER_EV: {
 488                 uint32_t id;
 489
 490                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 491
 492                 EFSYS_ASSERT(eecp->eec_timer != NULL);
 493                 should_abort = eecp->eec_timer(arg, id);
 494
 495                 break;
 496         }
 497         case FSE_AZ_RX_DSC_ERROR_EV:
 498                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
 499
 500                 EFSYS_PROBE(rx_dsc_error);
 501
 502                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 503                 should_abort = eecp->eec_exception(arg,
 504                         EFX_EXCEPTION_RX_DSC_ERROR, 0);
 505
 506                 break;
 507
 508         case FSE_AZ_TX_DSC_ERROR_EV:
 509                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
 510
 511                 EFSYS_PROBE(tx_dsc_error);
 512
 513                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 514                 should_abort = eecp->eec_exception(arg,
 515                         EFX_EXCEPTION_TX_DSC_ERROR, 0);
 516
 517                 break;
 518
 519         default:
 520                 break;
 521         }
 522
 523         return (should_abort);
 524 }
 525
 526 static  __checkReturn   boolean_t
 527 efx_ev_drv_gen(
 528         __in            efx_evq_t *eep,
 529         __in            efx_qword_t *eqp,
 530         __in            const efx_ev_callbacks_t *eecp,
 531         __in_opt        void *arg)
 532 {
 533         uint32_t data;
 534         boolean_t should_abort;
 535
 536         EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
 537
 538         data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
 539         if (data >= ((uint32_t)1 << 16)) {
 540                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 541                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 542                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 543                 return (B_TRUE);
 544         }
 545
 546         EFSYS_ASSERT(eecp->eec_software != NULL);
 547         should_abort = eecp->eec_software(arg, (uint16_t)data);
 548
 549         return (should_abort);
 550 }
 551
 552 #if EFSYS_OPT_MCDI
 553
 554 static  __checkReturn   boolean_t
 555 efx_ev_mcdi(
 556         __in            efx_evq_t *eep,
 557         __in            efx_qword_t *eqp,
 558         __in            const efx_ev_callbacks_t *eecp,
 559         __in_opt        void *arg)
 560 {
 561         efx_nic_t *enp = eep->ee_enp;
 562         unsigned code;
 563         boolean_t should_abort = B_FALSE;
 564
 565         EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
 566
 567         if (enp->en_family != EFX_FAMILY_SIENA)
 568                 goto out;
 569
 570         EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
 571
 572         code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
 573         switch (code) {
 574         case MCDI_EVENT_CODE_BADSSERT:
 575                 efx_mcdi_ev_death(enp, EINTR);
 576                 break;
 577
 578         case MCDI_EVENT_CODE_CMDDONE:
 579                 efx_mcdi_ev_cpl(enp,
 580                                 MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
 581                                 MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
 582                                 MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
 583                 break;
 584
 585         case MCDI_EVENT_CODE_LINKCHANGE: {
 586                 efx_link_mode_t link_mode;
 587
 588                 siena_phy_link_ev(enp, eqp, &link_mode);
 589                 should_abort = eecp->eec_link_change(arg, link_mode);
 590                 break;
 591         }
 592         case MCDI_EVENT_CODE_SENSOREVT: {
 593 #if EFSYS_OPT_MON_STATS
 594                 efx_mon_stat_t id;
 595                 efx_mon_stat_value_t value;
 596                 int rc;
 597
 598                 if ((rc = siena_mon_ev(enp, eqp, &id, &value)) == 0)
 599                         should_abort = eecp->eec_monitor(arg, id, value);
 600                 else if (rc == ENOTSUP) {
 601                         should_abort = eecp->eec_exception(arg,
 602                                 EFX_EXCEPTION_UNKNOWN_SENSOREVT,
 603                                 MCDI_EV_FIELD(eqp, DATA));
 604                 } else
 605                         EFSYS_ASSERT(rc == ENODEV);     /* Wrong port */
 606 #else
 607                 should_abort = B_FALSE;
 608 #endif
 609                 break;
 610         }
 611         case MCDI_EVENT_CODE_SCHEDERR:
 612                 /* Informational only */
 613                 break;
 614
 615         case MCDI_EVENT_CODE_REBOOT:
 616                 efx_mcdi_ev_death(enp, EIO);
 617                 break;
 618
 619         case MCDI_EVENT_CODE_MAC_STATS_DMA:
 620 #if EFSYS_OPT_MAC_STATS
 621                 if (eecp->eec_mac_stats != NULL) {
 622                         eecp->eec_mac_stats(arg,
 623                             MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
 624                 }
 625 #endif
 626                 break;
 627
 628         case MCDI_EVENT_CODE_FWALERT: {
 629                 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
 630
 631                 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
 632                         should_abort = eecp->eec_exception(arg,
 633                                 EFX_EXCEPTION_FWALERT_SRAM,
 634                                 MCDI_EV_FIELD(eqp, FWALERT_DATA));
 635                 else
 636                         should_abort = eecp->eec_exception(arg,
 637                                 EFX_EXCEPTION_UNKNOWN_FWALERT,
 638                                 MCDI_EV_FIELD(eqp, DATA));
 639                 break;
 640         }
 641
 642         default:
 643                 EFSYS_PROBE1(mc_pcol_error, int, code);
 644                 break;
 645         }
 646
 647 out:
 648         return (should_abort);
 649 }
 650
 651 #endif  /* EFSYS_OPT_SIENA */
 652
 653         __checkReturn   int
 654 efx_ev_qprime(
 655         __in            efx_evq_t *eep,
 656         __in            unsigned int count)
 657 {
 658         efx_nic_t *enp = eep->ee_enp;
 659         uint32_t rptr;
 660         efx_dword_t dword;
 661         int rc;
 662
 663         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 664
 665         if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
 666                 rc = EINVAL;
 667                 goto fail1;
 668         }
 669
 670         rptr = count & eep->ee_mask;
 671
 672         EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
 673
 674         EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
 675                             &dword, B_FALSE);
 676
 677         return (0);
 678
 679 fail1:
 680         EFSYS_PROBE1(fail1, int, rc);
 681
 682         return (rc);
 683 }
 684
 685         __checkReturn   boolean_t
 686 efx_ev_qpending(
 687         __in            efx_evq_t *eep,
 688         __in            unsigned int count)
 689 {
 690         size_t offset;
 691         efx_qword_t qword;
 692
 693         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 694
 695         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 696         EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
 697
 698         return (EFX_QWORD_FIELD(qword, EFX_DWORD_0) != 0xffffffff &&
 699                 EFX_QWORD_FIELD(qword, EFX_DWORD_1) != 0xffffffff);
 700 }
 701
 702 #if EFSYS_OPT_EV_PREFETCH
 703
 704                         void
 705 efx_ev_qprefetch(
 706         __in            efx_evq_t *eep,
 707         __in            unsigned int count)
 708 {
 709         unsigned int offset;
 710
 711         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 712
 713         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 714         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 715 }
 716
 717 #endif  /* EFSYS_OPT_EV_PREFETCH */
 718
 719 #define EFX_EV_BATCH    8
 720
 721 #define EFX_EV_PRESENT(_qword)                                          \
 722         (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&        \
 723         EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
 724
 725                         void
 726 efx_ev_qpoll(
 727         __in            efx_evq_t *eep,
 728         __inout         unsigned int *countp,
 729         __in            const efx_ev_callbacks_t *eecp,
 730         __in_opt        void *arg)
 731 {
 732         efx_qword_t ev[EFX_EV_BATCH];
 733         unsigned int batch;
 734         unsigned int total;
 735         unsigned int count;
 736         unsigned int index;
 737         size_t offset;
 738
 739         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 740         EFSYS_ASSERT(countp != NULL);
 741         EFSYS_ASSERT(eecp != NULL);
 742
 743         count = *countp;
 744         do {
 745                 /* Read up until the end of the batch period */
 746                 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
 747                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 748                 for (total = 0; total < batch; ++total) {
 749                         EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
 750
 751                         if (!EFX_EV_PRESENT(ev[total]))
 752                                 break;
 753
 754                         EFSYS_PROBE3(event, unsigned int, eep->ee_index,
 755                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
 756                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
 757
 758                         offset += sizeof (efx_qword_t);
 759                 }
 760
 761 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
 762                 /*
 763                  * Prefetch the next batch when we get within PREFETCH_PERIOD
 764                  * of a completed batch. If the batch is smaller, then prefetch
 765                  * immediately.
 766                  */
 767                 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
 768                         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 769 #endif  /* EFSYS_OPT_EV_PREFETCH */
 770
 771                 /* Process the batch of events */
 772                 for (index = 0; index < total; ++index) {
 773                         boolean_t should_abort;
 774                         uint32_t code;
 775                         efx_ev_handler_t handler;
 776
 777 #if EFSYS_OPT_EV_PREFETCH
 778                         /* Prefetch if we've now reached the batch period */
 779                         if (total == batch &&
 780                             index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
 781                                 offset = (count + batch) & eep->ee_mask;
 782                                 offset *= sizeof (efx_qword_t);
 783
 784                                 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 785                         }
 786 #endif  /* EFSYS_OPT_EV_PREFETCH */
 787
 788                         EFX_EV_QSTAT_INCR(eep, EV_ALL);
 789
 790                         code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
 791                         handler = eep->ee_handler[code];
 792                         EFSYS_ASSERT(handler != NULL);
 793                         should_abort = handler(eep, &(ev[index]), eecp, arg);
 794                         if (should_abort) {
 795                                 /* Ignore subsequent events */
 796                                 total = index + 1;
 797                                 break;
 798                         }
 799                 }
 800
 801                 /*
 802                  * Now that the hardware has most likely moved onto dma'ing
 803                  * into the next cache line, clear the processed events. Take
 804                  * care to only clear out events that we've processed
 805                  */
 806                 EFX_SET_QWORD(ev[0]);
 807                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 808                 for (index = 0; index < total; ++index) {
 809                         EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
 810                         offset += sizeof (efx_qword_t);
 811                 }
 812
 813                 count += total;
 814
 815         } while (total == batch);
 816
 817         *countp = count;
 818 }
 819
 820                 void
 821 efx_ev_qpost(
 822         __in    efx_evq_t *eep,
 823         __in    uint16_t data)
 824 {
 825         efx_nic_t *enp = eep->ee_enp;
 826         efx_qword_t ev;
 827         efx_oword_t oword;
 828
 829         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 830
 831         EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
 832             FSF_AZ_EV_DATA_DW0, (uint32_t)data);
 833
 834         EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
 835             EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
 836             EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
 837
 838         EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
 839 }
 840
 841         __checkReturn   int
 842 efx_ev_qmoderate(
 843         __in            efx_evq_t *eep,
 844         __in            unsigned int us)
 845 {
 846         efx_nic_t *enp = eep->ee_enp;
 847         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 848         unsigned int locked;
 849         efx_dword_t dword;
 850         int rc;
 851
 852         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 853
 854         if (us > encp->enc_evq_moderation_max) {
 855                 rc = EINVAL;
 856                 goto fail1;
 857         }
 858
 859         /* If the value is zero then disable the timer */
 860         if (us == 0) {
 861                 if (enp->en_family == EFX_FAMILY_FALCON)
 862                         EFX_POPULATE_DWORD_2(dword,
 863                             FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_DIS,
 864                             FRF_AB_TC_TIMER_VAL, 0);
 865                 else
 866                         EFX_POPULATE_DWORD_2(dword,
 867                             FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
 868                             FRF_CZ_TC_TIMER_VAL, 0);
 869         } else {
 870                 uint32_t timer_val;
 871
 872                 /* Calculate the timer value in quanta */
 873                 timer_val = us * encp->enc_clk_mult / EFX_EV_TIMER_QUANTUM;
 874
 875                 /* Moderation value is base 0 so we need to deduct 1 */
 876                 if (timer_val > 0)
 877                         timer_val--;
 878
 879                 if (enp->en_family == EFX_FAMILY_FALCON)
 880                         EFX_POPULATE_DWORD_2(dword,
 881                             FRF_AB_TC_TIMER_MODE, FFE_AB_TIMER_MODE_INT_HLDOFF,
 882                             FRF_AB_TIMER_VAL, timer_val);
 883                 else
 884                         EFX_POPULATE_DWORD_2(dword,
 885                             FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
 886                             FRF_CZ_TC_TIMER_VAL, timer_val);
 887         }
 888
 889         locked = (eep->ee_index == 0) ? 1 : 0;
 890
 891         EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
 892             eep->ee_index, &dword, locked);
 893
 894         return (0);
 895
 896 fail1:
 897         EFSYS_PROBE1(fail1, int, rc);
 898
 899         return (rc);
 900 }
 901
 902         __checkReturn   int
 903 efx_ev_qcreate(
 904         __in            efx_nic_t *enp,
 905         __in            unsigned int index,
 906         __in            efsys_mem_t *esmp,
 907         __in            size_t n,
 908         __in            uint32_t id,
 909         __deref_out     efx_evq_t **eepp)
 910 {
 911         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 912         uint32_t size;
 913         efx_evq_t *eep;
 914         efx_oword_t oword;
 915         int rc;
 916
 917         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 918         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 919
 920         EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
 921
 922         if (!ISP2(n) || !(n & EFX_EVQ_NEVS_MASK)) {
 923                 rc = EINVAL;
 924                 goto fail1;
 925         }
 926         if (index >= encp->enc_evq_limit) {
 927                 rc = EINVAL;
 928                 goto fail2;
 929         }
 930 #if EFSYS_OPT_RX_SCALE
 931         if (enp->en_intr.ei_type == EFX_INTR_LINE &&
 932             index >= EFX_MAXRSS_LEGACY) {
 933                 rc = EINVAL;
 934                 goto fail3;
 935         }
 936 #endif
 937         for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS);
 938             size++)
 939                 if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS))
 940                         break;
 941         if (id + (1 << size) >= encp->enc_buftbl_limit) {
 942                 rc = EINVAL;
 943                 goto fail4;
 944         }
 945
 946         /* Allocate an EVQ object */
 947         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
 948         if (eep == NULL) {
 949                 rc = ENOMEM;
 950                 goto fail5;
 951         }
 952
 953         eep->ee_magic = EFX_EVQ_MAGIC;
 954         eep->ee_enp = enp;
 955         eep->ee_index = index;
 956         eep->ee_mask = n - 1;
 957         eep->ee_esmp = esmp;
 958
 959         /* Set up the handler table */
 960         eep->ee_handler[FSE_AZ_EV_CODE_RX_EV] = efx_ev_rx;
 961         eep->ee_handler[FSE_AZ_EV_CODE_TX_EV] = efx_ev_tx;
 962         eep->ee_handler[FSE_AZ_EV_CODE_DRIVER_EV] = efx_ev_driver;
 963         eep->ee_handler[FSE_AZ_EV_CODE_GLOBAL_EV] = efx_ev_global;
 964         eep->ee_handler[FSE_AZ_EV_CODE_DRV_GEN_EV] = efx_ev_drv_gen;
 965 #if EFSYS_OPT_MCDI
 966         eep->ee_handler[FSE_AZ_EV_CODE_MCDI_EVRESPONSE] = efx_ev_mcdi;
 967 #endif  /* EFSYS_OPT_SIENA */
 968
 969         /* Set up the new event queue */
 970         if (enp->en_family != EFX_FAMILY_FALCON) {
 971                 EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1);
 972                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword);
 973         }
 974
 975         EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
 976             FRF_AZ_EVQ_BUF_BASE_ID, id);
 977
 978         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword);
 979
 980         enp->en_ev_qcount++;
 981         *eepp = eep;
 982         return (0);
 983
 984 fail5:
 985         EFSYS_PROBE(fail5);
 986 fail4:
 987         EFSYS_PROBE(fail4);
 988 #if EFSYS_OPT_RX_SCALE
 989 fail3:
 990         EFSYS_PROBE(fail3);
 991 #endif
 992 fail2:
 993         EFSYS_PROBE(fail2);
 994 fail1:
 995         EFSYS_PROBE1(fail1, int, rc);
 996
 997         return (rc);
 998 }
 999
1000 #if EFSYS_OPT_QSTATS
1001 #if EFSYS_OPT_NAMES
1002 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock 67e9bdcd920059bd */
1003 static const char       __cs * __cs __efx_ev_qstat_name[] = {
1004         "all",
1005         "rx",
1006         "rx_ok",
1007         "rx_recovery",
1008         "rx_frm_trunc",
1009         "rx_tobe_disc",
1010         "rx_pause_frm_err",
1011         "rx_buf_owner_id_err",
1012         "rx_ipv4_hdr_chksum_err",
1013         "rx_tcp_udp_chksum_err",
1014         "rx_eth_crc_err",
1015         "rx_ip_frag_err",
1016         "rx_mcast_pkt",
1017         "rx_mcast_hash_match",
1018         "rx_tcp_ipv4",
1019         "rx_tcp_ipv6",
1020         "rx_udp_ipv4",
1021         "rx_udp_ipv6",
1022         "rx_other_ipv4",
1023         "rx_other_ipv6",
1024         "rx_non_ip",
1025         "rx_overrun",
1026         "tx",
1027         "tx_wq_ff_full",
1028         "tx_pkt_err",
1029         "tx_pkt_too_big",
1030         "tx_unexpected",
1031         "global",
1032         "global_phy",
1033         "global_mnt",
1034         "global_rx_recovery",
1035         "driver",
1036         "driver_srm_upd_done",
1037         "driver_tx_descq_fls_done",
1038         "driver_rx_descq_fls_done",
1039         "driver_rx_descq_fls_failed",
1040         "driver_rx_dsc_error",
1041         "driver_tx_dsc_error",
1042         "drv_gen",
1043         "mcdi_response",
1044 };
1045 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1046
1047                 const char __cs *
1048 efx_ev_qstat_name(
1049         __in    efx_nic_t *enp,
1050         __in    unsigned int id)
1051 {
1052         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1053         EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1054
1055         return (__efx_ev_qstat_name[id]);
1056 }
1057 #endif  /* EFSYS_OPT_NAMES */
1058 #endif  /* EFSYS_OPT_QSTATS */
1059
1060 #if EFSYS_OPT_QSTATS
1061                                         void
1062 efx_ev_qstats_update(
1063         __in                            efx_evq_t *eep,
1064         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
1065 {
1066         unsigned int id;
1067
1068         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
1069
1070         for (id = 0; id < EV_NQSTATS; id++) {
1071                 efsys_stat_t *essp = &stat[id];
1072
1073                 EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1074                 eep->ee_stat[id] = 0;
1075         }
1076 }
1077 #endif  /* EFSYS_OPT_QSTATS */
1078
1079                 void
1080 efx_ev_qdestroy(
1081         __in    efx_evq_t *eep)
1082 {
1083         efx_nic_t *enp = eep->ee_enp;
1084         efx_oword_t oword;
1085
1086         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
1087
1088         EFSYS_ASSERT(enp->en_ev_qcount != 0);
1089         --enp->en_ev_qcount;
1090
1091         /* Purge event queue */
1092         EFX_ZERO_OWORD(oword);
1093
1094         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1095             eep->ee_index, &oword);
1096
1097         if (enp->en_family != EFX_FAMILY_FALCON) {
1098                 EFX_ZERO_OWORD(oword);
1099                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL,
1100                     eep->ee_index, &oword);
1101         }
1102
1103         /* Free the EVQ object */
1104         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
1105 }
1106
1107                 void
1108 efx_ev_fini(
1109         __in    efx_nic_t *enp)
1110 {
1111         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1112         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
1113         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
1114         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
1115         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
1116         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
1117
1118         enp->en_mod_flags &= ~EFX_MOD_EV;
1119 }