sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/kernel.h>
  35 #include <sys/malloc.h>
  36 #include <sys/param.h>
  37 #include <sys/queue.h>
  38 #include <sys/systm.h>
  39 #include <sys/taskqueue.h>
  40
  41 #include "common/efx.h"
  42
  43 #include "sfxge.h"
  44
  45 static void
  46 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  47 {
  48         struct sfxge_softc *sc;
  49         unsigned int index;
  50         struct sfxge_rxq *rxq;
  51         struct sfxge_txq *txq;
  52
  53         sc = evq->sc;
  54         index = evq->index;
  55         rxq = sc->rxq[index];
  56
  57         if ((txq = evq->txq) != NULL) {
  58                 evq->txq = NULL;
  59                 evq->txqs = &(evq->txq);
  60
  61                 do {
  62                         struct sfxge_txq *next;
  63
  64                         next = txq->next;
  65                         txq->next = NULL;
  66
  67                         KASSERT(txq->evq_index == index,
  68                             ("txq->evq_index != index"));
  69
  70                         if (txq->pending != txq->completed)
  71                                 sfxge_tx_qcomplete(txq);
  72
  73                         txq = next;
  74                 } while (txq != NULL);
  75         }
  76
  77         if (rxq->pending != rxq->completed)
  78                 sfxge_rx_qcomplete(rxq, eop);
  79 }
  80
  81 static boolean_t
  82 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  83     uint16_t flags)
  84 {
  85         struct sfxge_evq *evq;
  86         struct sfxge_softc *sc;
  87         struct sfxge_rxq *rxq;
  88         unsigned int expected;
  89         struct sfxge_rx_sw_desc *rx_desc;
  90
  91         evq = arg;
  92         sc = evq->sc;
  93
  94         if (evq->exception)
  95                 goto done;
  96
  97         rxq = sc->rxq[label];
  98         KASSERT(rxq != NULL, ("rxq == NULL"));
  99         KASSERT(evq->index == rxq->index,
 100             ("evq->index != rxq->index"));
 101
 102         if (rxq->init_state != SFXGE_RXQ_STARTED)
 103                 goto done;
 104
 105         expected = rxq->pending++ & (SFXGE_NDESCS - 1);
 106         if (id != expected) {
 107                 evq->exception = B_TRUE;
 108
 109                 device_printf(sc->dev, "RX completion out of order"
 110                               " (id=%#x expected=%#x flags=%#x); resetting\n",
 111                               id, expected, flags);
 112                 sfxge_schedule_reset(sc);
 113
 114                 goto done;
 115         }
 116
 117         rx_desc = &rxq->queue[id];
 118
 119         KASSERT(rx_desc->flags == EFX_DISCARD,
 120             ("rx_desc->flags != EFX_DISCARD"));
 121         rx_desc->flags = flags;
 122
 123         KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 124         rx_desc->size = (uint16_t)size;
 125         prefetch_read_many(rx_desc->mbuf);
 126
 127         evq->rx_done++;
 128
 129         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 130                 sfxge_ev_qcomplete(evq, B_FALSE);
 131
 132 done:
 133         return (evq->rx_done >= SFXGE_EV_BATCH);
 134 }
 135
 136 static boolean_t
 137 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 138 {
 139         struct sfxge_evq *evq;
 140         struct sfxge_softc *sc;
 141
 142         evq = (struct sfxge_evq *)arg;
 143         sc = evq->sc;
 144
 145         evq->exception = B_TRUE;
 146
 147         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 148                 device_printf(sc->dev,
 149                               "hardware exception (code=%u); resetting\n",
 150                               code);
 151                 sfxge_schedule_reset(sc);
 152         }
 153
 154         return (B_FALSE);
 155 }
 156
 157 static boolean_t
 158 sfxge_ev_rxq_flush_done(void *arg, uint32_t label)
 159 {
 160         struct sfxge_evq *evq;
 161         struct sfxge_softc *sc;
 162         struct sfxge_rxq *rxq;
 163         unsigned int index;
 164         uint16_t magic;
 165
 166         evq = (struct sfxge_evq *)arg;
 167         sc = evq->sc;
 168         rxq = sc->rxq[label];
 169
 170         KASSERT(rxq != NULL, ("rxq == NULL"));
 171
 172         /* Resend a software event on the correct queue */
 173         index = rxq->index;
 174         evq = sc->evq[index];
 175
 176         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 177             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
 178         magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
 179
 180         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 181             ("evq not started"));
 182         efx_ev_qpost(evq->common, magic);
 183
 184         return (B_FALSE);
 185 }
 186
 187 static boolean_t
 188 sfxge_ev_rxq_flush_failed(void *arg, uint32_t label)
 189 {
 190         struct sfxge_evq *evq;
 191         struct sfxge_softc *sc;
 192         struct sfxge_rxq *rxq;
 193         unsigned int index;
 194         uint16_t magic;
 195
 196         evq = (struct sfxge_evq *)arg;
 197         sc = evq->sc;
 198         rxq = sc->rxq[label];
 199
 200         KASSERT(rxq != NULL, ("rxq == NULL"));
 201
 202         /* Resend a software event on the correct queue */
 203         index = rxq->index;
 204         evq = sc->evq[index];
 205
 206         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 207             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 208         magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
 209
 210         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 211             ("evq not started"));
 212         efx_ev_qpost(evq->common, magic);
 213
 214         return (B_FALSE);
 215 }
 216
 217 static boolean_t
 218 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 219 {
 220         struct sfxge_evq *evq;
 221         struct sfxge_softc *sc;
 222         struct sfxge_txq *txq;
 223         unsigned int stop;
 224         unsigned int delta;
 225
 226         evq = (struct sfxge_evq *)arg;
 227         sc = evq->sc;
 228         txq = sc->txq[label];
 229
 230         KASSERT(txq != NULL, ("txq == NULL"));
 231         KASSERT(evq->index == txq->evq_index,
 232             ("evq->index != txq->evq_index"));
 233
 234         if (txq->init_state != SFXGE_TXQ_STARTED)
 235                 goto done;
 236
 237         stop = (id + 1) & (SFXGE_NDESCS - 1);
 238         id = txq->pending & (SFXGE_NDESCS - 1);
 239
 240         delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop);
 241         txq->pending += delta;
 242
 243         evq->tx_done++;
 244
 245         if (txq->next == NULL &&
 246             evq->txqs != &(txq->next)) {
 247                 *(evq->txqs) = txq;
 248                 evq->txqs = &(txq->next);
 249         }
 250
 251         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 252                 sfxge_tx_qcomplete(txq);
 253
 254 done:
 255         return (evq->tx_done >= SFXGE_EV_BATCH);
 256 }
 257
 258 static boolean_t
 259 sfxge_ev_txq_flush_done(void *arg, uint32_t label)
 260 {
 261         struct sfxge_evq *evq;
 262         struct sfxge_softc *sc;
 263         struct sfxge_txq *txq;
 264         uint16_t magic;
 265
 266         evq = (struct sfxge_evq *)arg;
 267         sc = evq->sc;
 268         txq = sc->txq[label];
 269
 270         KASSERT(txq != NULL, ("txq == NULL"));
 271         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 272             ("txq not initialized"));
 273
 274         /* Resend a software event on the correct queue */
 275         evq = sc->evq[txq->evq_index];
 276
 277         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 278             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 279         magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
 280
 281         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 282             ("evq not started"));
 283         efx_ev_qpost(evq->common, magic);
 284
 285         return (B_FALSE);
 286 }
 287
 288 static boolean_t
 289 sfxge_ev_software(void *arg, uint16_t magic)
 290 {
 291         struct sfxge_evq *evq;
 292         struct sfxge_softc *sc;
 293         unsigned int label;
 294
 295         evq = (struct sfxge_evq *)arg;
 296         sc = evq->sc;
 297
 298         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 299         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 300
 301         switch (magic) {
 302         case SFXGE_MAGIC_RX_QFLUSH_DONE: {
 303                 struct sfxge_rxq *rxq = sc->rxq[label];
 304
 305                 KASSERT(rxq != NULL, ("rxq == NULL"));
 306                 KASSERT(evq->index == rxq->index,
 307                     ("evq->index != rxq->index"));
 308
 309                 sfxge_rx_qflush_done(rxq);
 310                 break;
 311         }
 312         case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
 313                 struct sfxge_rxq *rxq = sc->rxq[label];
 314
 315                 KASSERT(rxq != NULL, ("rxq == NULL"));
 316                 KASSERT(evq->index == rxq->index,
 317                     ("evq->index != rxq->index"));
 318
 319                 sfxge_rx_qflush_failed(rxq);
 320                 break;
 321         }
 322         case SFXGE_MAGIC_RX_QREFILL: {
 323                 struct sfxge_rxq *rxq = sc->rxq[label];
 324
 325                 KASSERT(rxq != NULL, ("rxq == NULL"));
 326                 KASSERT(evq->index == rxq->index,
 327                     ("evq->index != rxq->index"));
 328
 329                 sfxge_rx_qrefill(rxq);
 330                 break;
 331         }
 332         case SFXGE_MAGIC_TX_QFLUSH_DONE: {
 333                 struct sfxge_txq *txq = sc->txq[label];
 334
 335                 KASSERT(txq != NULL, ("txq == NULL"));
 336                 KASSERT(evq->index == txq->evq_index,
 337                     ("evq->index != txq->evq_index"));
 338
 339                 sfxge_tx_qflush_done(txq);
 340                 break;
 341         }
 342         default:
 343                 break;
 344         }
 345
 346         return (B_FALSE);
 347 }
 348
 349 static boolean_t
 350 sfxge_ev_sram(void *arg, uint32_t code)
 351 {
 352         (void)arg;
 353         (void)code;
 354
 355         switch (code) {
 356         case EFX_SRAM_UPDATE:
 357                 EFSYS_PROBE(sram_update);
 358                 break;
 359
 360         case EFX_SRAM_CLEAR:
 361                 EFSYS_PROBE(sram_clear);
 362                 break;
 363
 364         case EFX_SRAM_ILLEGAL_CLEAR:
 365                 EFSYS_PROBE(sram_illegal_clear);
 366                 break;
 367
 368         default:
 369                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 370                 break;
 371         }
 372
 373         return (B_FALSE);
 374 }
 375
 376 static boolean_t
 377 sfxge_ev_timer(void *arg, uint32_t index)
 378 {
 379         (void)arg;
 380         (void)index;
 381
 382         return (B_FALSE);
 383 }
 384
 385 static boolean_t
 386 sfxge_ev_wake_up(void *arg, uint32_t index)
 387 {
 388         (void)arg;
 389         (void)index;
 390
 391         return (B_FALSE);
 392 }
 393
 394 static void
 395 sfxge_ev_stat_update(struct sfxge_softc *sc)
 396 {
 397         struct sfxge_evq *evq;
 398         unsigned int index;
 399         clock_t now;
 400
 401         sx_xlock(&sc->softc_lock);
 402
 403         if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
 404                 goto out;
 405
 406         now = ticks;
 407         if (now - sc->ev_stats_update_time < hz)
 408                 goto out;
 409
 410         sc->ev_stats_update_time = now;
 411
 412         /* Add event counts from each event queue in turn */
 413         for (index = 0; index < sc->intr.n_alloc; index++) {
 414                 evq = sc->evq[index];
 415                 mtx_lock(&evq->lock);
 416                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 417                 mtx_unlock(&evq->lock);
 418         }
 419 out:
 420         sx_xunlock(&sc->softc_lock);
 421 }
 422
 423 static int
 424 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 425 {
 426         struct sfxge_softc *sc = arg1;
 427         unsigned int id = arg2;
 428
 429         sfxge_ev_stat_update(sc);
 430
 431         return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]));
 432 }
 433
 434 static void
 435 sfxge_ev_stat_init(struct sfxge_softc *sc)
 436 {
 437         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 438         struct sysctl_oid_list *stat_list;
 439         unsigned int id;
 440         char name[40];
 441
 442         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 443
 444         for (id = 0; id < EV_NQSTATS; id++) {
 445                 snprintf(name, sizeof(name), "ev_%s",
 446                          efx_ev_qstat_name(sc->enp, id));
 447                 SYSCTL_ADD_PROC(
 448                         ctx, stat_list,
 449                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 450                         sc, id, sfxge_ev_stat_handler, "Q",
 451                         "");
 452         }
 453 }
 454
 455 static void
 456 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 457 {
 458         struct sfxge_evq *evq;
 459         efx_evq_t *eep;
 460
 461         evq = sc->evq[idx];
 462         eep = evq->common;
 463
 464         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 465             ("evq->init_state != SFXGE_EVQ_STARTED"));
 466
 467         (void)efx_ev_qmoderate(eep, us);
 468 }
 469
 470 static int
 471 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 472 {
 473         struct sfxge_softc *sc = arg1;
 474         struct sfxge_intr *intr = &sc->intr;
 475         unsigned int moderation;
 476         int error;
 477         int index;
 478
 479         sx_xlock(&sc->softc_lock);
 480
 481         if (req->newptr) {
 482                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 483                     != 0)
 484                         goto out;
 485
 486                 /* We may not be calling efx_ev_qmoderate() now,
 487                  * so we have to range-check the value ourselves.
 488                  */
 489                 if (moderation >
 490                     efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
 491                         error = EINVAL;
 492                         goto out;
 493                 }
 494
 495                 sc->ev_moderation = moderation;
 496                 if (intr->state == SFXGE_INTR_STARTED) {
 497                         for (index = 0; index < intr->n_alloc; index++)
 498                                 sfxge_ev_qmoderate(sc, index, moderation);
 499                 }
 500         } else {
 501                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 502                                    sizeof(sc->ev_moderation));
 503         }
 504
 505 out:
 506         sx_xunlock(&sc->softc_lock);
 507
 508         return error;
 509 }
 510
 511 static boolean_t
 512 sfxge_ev_initialized(void *arg)
 513 {
 514         struct sfxge_evq *evq;
 515
 516         evq = (struct sfxge_evq *)arg;
 517
 518         KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
 519             ("evq not starting"));
 520
 521         evq->init_state = SFXGE_EVQ_STARTED;
 522
 523         return (0);
 524 }
 525
 526 static boolean_t
 527 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 528 {
 529         struct sfxge_evq *evq;
 530         struct sfxge_softc *sc;
 531
 532         evq = (struct sfxge_evq *)arg;
 533         sc = evq->sc;
 534
 535         sfxge_mac_link_update(sc, link_mode);
 536
 537         return (0);
 538 }
 539
 540 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 541         .eec_initialized        = sfxge_ev_initialized,
 542         .eec_rx                 = sfxge_ev_rx,
 543         .eec_tx                 = sfxge_ev_tx,
 544         .eec_exception          = sfxge_ev_exception,
 545         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 546         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 547         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 548         .eec_software           = sfxge_ev_software,
 549         .eec_sram               = sfxge_ev_sram,
 550         .eec_wake_up            = sfxge_ev_wake_up,
 551         .eec_timer              = sfxge_ev_timer,
 552         .eec_link_change        = sfxge_ev_link_change,
 553 };
 554
 555
 556 int
 557 sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index)
 558 {
 559         struct sfxge_evq *evq;
 560         int rc;
 561
 562         evq = sc->evq[index];
 563
 564         mtx_lock(&evq->lock);
 565
 566         if (evq->init_state != SFXGE_EVQ_STARTING &&
 567             evq->init_state != SFXGE_EVQ_STARTED) {
 568                 rc = EINVAL;
 569                 goto fail;
 570         }
 571
 572         /* Synchronize the DMA memory for reading */
 573         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 574             BUS_DMASYNC_POSTREAD);
 575
 576         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 577         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 578         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 579         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 580
 581         /* Poll the queue */
 582         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 583
 584         evq->rx_done = 0;
 585         evq->tx_done = 0;
 586
 587         /* Perform any pending completion processing */
 588         sfxge_ev_qcomplete(evq, B_TRUE);
 589
 590         /* Re-prime the event queue for interrupts */
 591         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 592                 goto fail;
 593
 594         mtx_unlock(&evq->lock);
 595
 596         return (0);
 597
 598 fail:
 599         mtx_unlock(&(evq->lock));
 600         return (rc);
 601 }
 602
 603 static void
 604 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 605 {
 606         struct sfxge_evq *evq;
 607
 608         evq = sc->evq[index];
 609
 610         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 611             ("evq->init_state != SFXGE_EVQ_STARTED"));
 612
 613         mtx_lock(&evq->lock);
 614         evq->init_state = SFXGE_EVQ_INITIALIZED;
 615         evq->read_ptr = 0;
 616         evq->exception = B_FALSE;
 617
 618         /* Add event counts before discarding the common evq state */
 619         efx_ev_qstats_update(evq->common, sc->ev_stats);
 620
 621         efx_ev_qdestroy(evq->common);
 622         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 623             EFX_EVQ_NBUFS(SFXGE_NEVS));
 624         mtx_unlock(&evq->lock);
 625 }
 626
 627 static int
 628 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 629 {
 630         struct sfxge_evq *evq;
 631         efsys_mem_t *esmp;
 632         int count;
 633         int rc;
 634
 635         evq = sc->evq[index];
 636         esmp = &evq->mem;
 637
 638         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 639             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 640
 641         /* Clear all events. */
 642         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS));
 643
 644         /* Program the buffer table. */
 645         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 646             EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0)
 647                 return rc;
 648
 649         /* Create the common code event queue. */
 650         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS,
 651             evq->buf_base_id, &evq->common)) != 0)
 652                 goto fail;
 653
 654         mtx_lock(&evq->lock);
 655
 656         /* Set the default moderation */
 657         (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
 658
 659         /* Prime the event queue for interrupts */
 660         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 661                 goto fail2;
 662
 663         evq->init_state = SFXGE_EVQ_STARTING;
 664
 665         mtx_unlock(&evq->lock);
 666
 667         /* Wait for the initialization event */
 668         count = 0;
 669         do {
 670                 /* Pause for 100 ms */
 671                 pause("sfxge evq init", hz / 10);
 672
 673                 /* Check to see if the test event has been processed */
 674                 if (evq->init_state == SFXGE_EVQ_STARTED)
 675                         goto done;
 676
 677         } while (++count < 20);
 678
 679         rc = ETIMEDOUT;
 680         goto fail3;
 681
 682 done:
 683         return (0);
 684
 685 fail3:
 686         mtx_lock(&evq->lock);
 687         evq->init_state = SFXGE_EVQ_INITIALIZED;
 688 fail2:
 689         mtx_unlock(&evq->lock);
 690         efx_ev_qdestroy(evq->common);
 691 fail:
 692         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 693             EFX_EVQ_NBUFS(SFXGE_NEVS));
 694
 695         return (rc);
 696 }
 697
 698 void
 699 sfxge_ev_stop(struct sfxge_softc *sc)
 700 {
 701         struct sfxge_intr *intr;
 702         efx_nic_t *enp;
 703         int index;
 704
 705         intr = &sc->intr;
 706         enp = sc->enp;
 707
 708         KASSERT(intr->state == SFXGE_INTR_STARTED,
 709             ("Interrupts not started"));
 710
 711         /* Stop the event queue(s) */
 712         index = intr->n_alloc;
 713         while (--index >= 0)
 714                 sfxge_ev_qstop(sc, index);
 715
 716         /* Tear down the event module */
 717         efx_ev_fini(enp);
 718 }
 719
 720 int
 721 sfxge_ev_start(struct sfxge_softc *sc)
 722 {
 723         struct sfxge_intr *intr;
 724         int index;
 725         int rc;
 726
 727         intr = &sc->intr;
 728
 729         KASSERT(intr->state == SFXGE_INTR_STARTED,
 730             ("intr->state != SFXGE_INTR_STARTED"));
 731
 732         /* Initialize the event module */
 733         if ((rc = efx_ev_init(sc->enp)) != 0)
 734                 return rc;
 735
 736         /* Start the event queues */
 737         for (index = 0; index < intr->n_alloc; index++) {
 738                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 739                         goto fail;
 740         }
 741
 742         return (0);
 743
 744 fail:
 745         /* Stop the event queue(s) */
 746         while (--index >= 0)
 747                 sfxge_ev_qstop(sc, index);
 748
 749         /* Tear down the event module */
 750         efx_ev_fini(sc->enp);
 751
 752         return (rc);
 753 }
 754
 755 static void
 756 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 757 {
 758         struct sfxge_evq *evq;
 759
 760         evq = sc->evq[index];
 761
 762         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 763             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 764         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 765
 766         sfxge_dma_free(&evq->mem);
 767
 768         sc->evq[index] = NULL;
 769
 770         mtx_destroy(&evq->lock);
 771
 772         free(evq, M_SFXGE);
 773 }
 774
 775 static int
 776 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 777 {
 778         struct sfxge_evq *evq;
 779         efsys_mem_t *esmp;
 780         int rc;
 781
 782         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 783
 784         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 785         evq->sc = sc;
 786         evq->index = index;
 787         sc->evq[index] = evq;
 788         esmp = &evq->mem;
 789
 790         /* Initialise TX completion list */
 791         evq->txqs = &evq->txq;
 792
 793         /* Allocate DMA space. */
 794         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0)
 795                 return (rc);
 796
 797         /* Allocate buffer table entries. */
 798         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS),
 799                                  &evq->buf_base_id);
 800
 801         mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
 802
 803         evq->init_state = SFXGE_EVQ_INITIALIZED;
 804
 805         return (0);
 806 }
 807
 808 void
 809 sfxge_ev_fini(struct sfxge_softc *sc)
 810 {
 811         struct sfxge_intr *intr;
 812         int index;
 813
 814         intr = &sc->intr;
 815
 816         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 817             ("intr->state != SFXGE_INTR_INITIALIZED"));
 818
 819         sc->ev_moderation = 0;
 820
 821         /* Tear down the event queue(s). */
 822         index = intr->n_alloc;
 823         while (--index >= 0)
 824                 sfxge_ev_qfini(sc, index);
 825 }
 826
 827 int
 828 sfxge_ev_init(struct sfxge_softc *sc)
 829 {
 830         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 831         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 832         struct sfxge_intr *intr;
 833         int index;
 834         int rc;
 835
 836         intr = &sc->intr;
 837
 838         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 839             ("intr->state != SFXGE_INTR_INITIALIZED"));
 840
 841         /* Set default interrupt moderation; add a sysctl to
 842          * read and change it.
 843          */
 844         sc->ev_moderation = 30;
 845         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 846                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 847                         sc, 0, sfxge_int_mod_handler, "IU",
 848                         "sfxge interrupt moderation (us)");
 849
 850         /*
 851          * Initialize the event queue(s) - one per interrupt.
 852          */
 853         for (index = 0; index < intr->n_alloc; index++) {
 854                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 855                         goto fail;
 856         }
 857
 858         sfxge_ev_stat_init(sc);
 859
 860         return (0);
 861
 862 fail:
 863         while (--index >= 0)
 864                 sfxge_ev_qfini(sc, index);
 865
 866         return (rc);
 867 }