sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35
  36 #include "common/efx.h"
  37
  38 #include "sfxge.h"
  39
  40 static void
  41 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  42 {
  43         struct sfxge_softc *sc;
  44         unsigned int index;
  45         struct sfxge_rxq *rxq;
  46         struct sfxge_txq *txq;
  47
  48         sc = evq->sc;
  49         index = evq->index;
  50         rxq = sc->rxq[index];
  51
  52         if ((txq = evq->txq) != NULL) {
  53                 evq->txq = NULL;
  54                 evq->txqs = &(evq->txq);
  55
  56                 do {
  57                         struct sfxge_txq *next;
  58
  59                         next = txq->next;
  60                         txq->next = NULL;
  61
  62                         KASSERT(txq->evq_index == index,
  63                             ("txq->evq_index != index"));
  64
  65                         if (txq->pending != txq->completed)
  66                                 sfxge_tx_qcomplete(txq);
  67
  68                         txq = next;
  69                 } while (txq != NULL);
  70         }
  71
  72         if (rxq->pending != rxq->completed)
  73                 sfxge_rx_qcomplete(rxq, eop);
  74 }
  75
  76 static boolean_t
  77 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  78     uint16_t flags)
  79 {
  80         struct sfxge_evq *evq;
  81         struct sfxge_softc *sc;
  82         struct sfxge_rxq *rxq;
  83         unsigned int expected;
  84         struct sfxge_rx_sw_desc *rx_desc;
  85
  86         evq = arg;
  87         sc = evq->sc;
  88
  89         if (evq->exception)
  90                 goto done;
  91
  92         rxq = sc->rxq[label];
  93         KASSERT(rxq != NULL, ("rxq == NULL"));
  94         KASSERT(evq->index == rxq->index,
  95             ("evq->index != rxq->index"));
  96
  97         if (rxq->init_state != SFXGE_RXQ_STARTED)
  98                 goto done;
  99
 100         expected = rxq->pending++ & (SFXGE_NDESCS - 1);
 101         if (id != expected) {
 102                 evq->exception = B_TRUE;
 103
 104                 device_printf(sc->dev, "RX completion out of order"
 105                               " (id=%#x expected=%#x flags=%#x); resetting\n",
 106                               id, expected, flags);
 107                 sfxge_schedule_reset(sc);
 108
 109                 goto done;
 110         }
 111
 112         rx_desc = &rxq->queue[id];
 113
 114         KASSERT(rx_desc->flags == EFX_DISCARD,
 115             ("rx_desc->flags != EFX_DISCARD"));
 116         rx_desc->flags = flags;
 117
 118         KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 119         rx_desc->size = (uint16_t)size;
 120         prefetch_read_many(rx_desc->mbuf);
 121
 122         evq->rx_done++;
 123
 124         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 125                 sfxge_ev_qcomplete(evq, B_FALSE);
 126
 127 done:
 128         return (evq->rx_done >= SFXGE_EV_BATCH);
 129 }
 130
 131 static boolean_t
 132 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 133 {
 134         struct sfxge_evq *evq;
 135         struct sfxge_softc *sc;
 136
 137         evq = (struct sfxge_evq *)arg;
 138         sc = evq->sc;
 139
 140         evq->exception = B_TRUE;
 141
 142         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 143                 device_printf(sc->dev,
 144                               "hardware exception (code=%u); resetting\n",
 145                               code);
 146                 sfxge_schedule_reset(sc);
 147         }
 148
 149         return (B_FALSE);
 150 }
 151
 152 static boolean_t
 153 sfxge_ev_rxq_flush_done(void *arg, uint32_t label)
 154 {
 155         struct sfxge_evq *evq;
 156         struct sfxge_softc *sc;
 157         struct sfxge_rxq *rxq;
 158         unsigned int index;
 159         uint16_t magic;
 160
 161         evq = (struct sfxge_evq *)arg;
 162         sc = evq->sc;
 163         rxq = sc->rxq[label];
 164
 165         KASSERT(rxq != NULL, ("rxq == NULL"));
 166
 167         /* Resend a software event on the correct queue */
 168         index = rxq->index;
 169         evq = sc->evq[index];
 170
 171         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 172             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
 173         magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
 174
 175         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 176             ("evq not started"));
 177         efx_ev_qpost(evq->common, magic);
 178
 179         return (B_FALSE);
 180 }
 181
 182 static boolean_t
 183 sfxge_ev_rxq_flush_failed(void *arg, uint32_t label)
 184 {
 185         struct sfxge_evq *evq;
 186         struct sfxge_softc *sc;
 187         struct sfxge_rxq *rxq;
 188         unsigned int index;
 189         uint16_t magic;
 190
 191         evq = (struct sfxge_evq *)arg;
 192         sc = evq->sc;
 193         rxq = sc->rxq[label];
 194
 195         KASSERT(rxq != NULL, ("rxq == NULL"));
 196
 197         /* Resend a software event on the correct queue */
 198         index = rxq->index;
 199         evq = sc->evq[index];
 200
 201         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 202             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 203         magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
 204
 205         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 206             ("evq not started"));
 207         efx_ev_qpost(evq->common, magic);
 208
 209         return (B_FALSE);
 210 }
 211
 212 static boolean_t
 213 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 214 {
 215         struct sfxge_evq *evq;
 216         struct sfxge_softc *sc;
 217         struct sfxge_txq *txq;
 218         unsigned int stop;
 219         unsigned int delta;
 220
 221         evq = (struct sfxge_evq *)arg;
 222         sc = evq->sc;
 223         txq = sc->txq[label];
 224
 225         KASSERT(txq != NULL, ("txq == NULL"));
 226         KASSERT(evq->index == txq->evq_index,
 227             ("evq->index != txq->evq_index"));
 228
 229         if (txq->init_state != SFXGE_TXQ_STARTED)
 230                 goto done;
 231
 232         stop = (id + 1) & (SFXGE_NDESCS - 1);
 233         id = txq->pending & (SFXGE_NDESCS - 1);
 234
 235         delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop);
 236         txq->pending += delta;
 237
 238         evq->tx_done++;
 239
 240         if (txq->next == NULL &&
 241             evq->txqs != &(txq->next)) {
 242                 *(evq->txqs) = txq;
 243                 evq->txqs = &(txq->next);
 244         }
 245
 246         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 247                 sfxge_tx_qcomplete(txq);
 248
 249 done:
 250         return (evq->tx_done >= SFXGE_EV_BATCH);
 251 }
 252
 253 static boolean_t
 254 sfxge_ev_txq_flush_done(void *arg, uint32_t label)
 255 {
 256         struct sfxge_evq *evq;
 257         struct sfxge_softc *sc;
 258         struct sfxge_txq *txq;
 259         uint16_t magic;
 260
 261         evq = (struct sfxge_evq *)arg;
 262         sc = evq->sc;
 263         txq = sc->txq[label];
 264
 265         KASSERT(txq != NULL, ("txq == NULL"));
 266         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 267             ("txq not initialized"));
 268
 269         /* Resend a software event on the correct queue */
 270         evq = sc->evq[txq->evq_index];
 271
 272         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 273             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 274         magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
 275
 276         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 277             ("evq not started"));
 278         efx_ev_qpost(evq->common, magic);
 279
 280         return (B_FALSE);
 281 }
 282
 283 static boolean_t
 284 sfxge_ev_software(void *arg, uint16_t magic)
 285 {
 286         struct sfxge_evq *evq;
 287         struct sfxge_softc *sc;
 288         unsigned int label;
 289
 290         evq = (struct sfxge_evq *)arg;
 291         sc = evq->sc;
 292
 293         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 294         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 295
 296         switch (magic) {
 297         case SFXGE_MAGIC_RX_QFLUSH_DONE: {
 298                 struct sfxge_rxq *rxq = sc->rxq[label];
 299
 300                 KASSERT(rxq != NULL, ("rxq == NULL"));
 301                 KASSERT(evq->index == rxq->index,
 302                     ("evq->index != rxq->index"));
 303
 304                 sfxge_rx_qflush_done(rxq);
 305                 break;
 306         }
 307         case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
 308                 struct sfxge_rxq *rxq = sc->rxq[label];
 309
 310                 KASSERT(rxq != NULL, ("rxq == NULL"));
 311                 KASSERT(evq->index == rxq->index,
 312                     ("evq->index != rxq->index"));
 313
 314                 sfxge_rx_qflush_failed(rxq);
 315                 break;
 316         }
 317         case SFXGE_MAGIC_RX_QREFILL: {
 318                 struct sfxge_rxq *rxq = sc->rxq[label];
 319
 320                 KASSERT(rxq != NULL, ("rxq == NULL"));
 321                 KASSERT(evq->index == rxq->index,
 322                     ("evq->index != rxq->index"));
 323
 324                 sfxge_rx_qrefill(rxq);
 325                 break;
 326         }
 327         case SFXGE_MAGIC_TX_QFLUSH_DONE: {
 328                 struct sfxge_txq *txq = sc->txq[label];
 329
 330                 KASSERT(txq != NULL, ("txq == NULL"));
 331                 KASSERT(evq->index == txq->evq_index,
 332                     ("evq->index != txq->evq_index"));
 333
 334                 sfxge_tx_qflush_done(txq);
 335                 break;
 336         }
 337         default:
 338                 break;
 339         }
 340
 341         return (B_FALSE);
 342 }
 343
 344 static boolean_t
 345 sfxge_ev_sram(void *arg, uint32_t code)
 346 {
 347         (void)arg;
 348         (void)code;
 349
 350         switch (code) {
 351         case EFX_SRAM_UPDATE:
 352                 EFSYS_PROBE(sram_update);
 353                 break;
 354
 355         case EFX_SRAM_CLEAR:
 356                 EFSYS_PROBE(sram_clear);
 357                 break;
 358
 359         case EFX_SRAM_ILLEGAL_CLEAR:
 360                 EFSYS_PROBE(sram_illegal_clear);
 361                 break;
 362
 363         default:
 364                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 365                 break;
 366         }
 367
 368         return (B_FALSE);
 369 }
 370
 371 static boolean_t
 372 sfxge_ev_timer(void *arg, uint32_t index)
 373 {
 374         (void)arg;
 375         (void)index;
 376
 377         return (B_FALSE);
 378 }
 379
 380 static boolean_t
 381 sfxge_ev_wake_up(void *arg, uint32_t index)
 382 {
 383         (void)arg;
 384         (void)index;
 385
 386         return (B_FALSE);
 387 }
 388
 389 static void
 390 sfxge_ev_stat_update(struct sfxge_softc *sc)
 391 {
 392         struct sfxge_evq *evq;
 393         unsigned int index;
 394         clock_t now;
 395
 396         sx_xlock(&sc->softc_lock);
 397
 398         if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
 399                 goto out;
 400
 401         now = ticks;
 402         if (now - sc->ev_stats_update_time < hz)
 403                 goto out;
 404
 405         sc->ev_stats_update_time = now;
 406
 407         /* Add event counts from each event queue in turn */
 408         for (index = 0; index < sc->intr.n_alloc; index++) {
 409                 evq = sc->evq[index];
 410                 mtx_lock(&evq->lock);
 411                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 412                 mtx_unlock(&evq->lock);
 413         }
 414 out:
 415         sx_xunlock(&sc->softc_lock);
 416 }
 417
 418 static int
 419 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 420 {
 421         struct sfxge_softc *sc = arg1;
 422         unsigned int id = arg2;
 423
 424         sfxge_ev_stat_update(sc);
 425
 426         return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]));
 427 }
 428
 429 static void
 430 sfxge_ev_stat_init(struct sfxge_softc *sc)
 431 {
 432         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 433         struct sysctl_oid_list *stat_list;
 434         unsigned int id;
 435         char name[40];
 436
 437         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 438
 439         for (id = 0; id < EV_NQSTATS; id++) {
 440                 snprintf(name, sizeof(name), "ev_%s",
 441                          efx_ev_qstat_name(sc->enp, id));
 442                 SYSCTL_ADD_PROC(
 443                         ctx, stat_list,
 444                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 445                         sc, id, sfxge_ev_stat_handler, "Q",
 446                         "");
 447         }
 448 }
 449
 450 static void
 451 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 452 {
 453         struct sfxge_evq *evq;
 454         efx_evq_t *eep;
 455
 456         evq = sc->evq[idx];
 457         eep = evq->common;
 458
 459         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 460             ("evq->init_state != SFXGE_EVQ_STARTED"));
 461
 462         (void)efx_ev_qmoderate(eep, us);
 463 }
 464
 465 static int
 466 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 467 {
 468         struct sfxge_softc *sc = arg1;
 469         struct sfxge_intr *intr = &sc->intr;
 470         unsigned int moderation;
 471         int error;
 472         int index;
 473
 474         sx_xlock(&sc->softc_lock);
 475
 476         if (req->newptr) {
 477                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 478                     != 0)
 479                         goto out;
 480
 481                 /* We may not be calling efx_ev_qmoderate() now,
 482                  * so we have to range-check the value ourselves.
 483                  */
 484                 if (moderation >
 485                     efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
 486                         error = EINVAL;
 487                         goto out;
 488                 }
 489
 490                 sc->ev_moderation = moderation;
 491                 if (intr->state == SFXGE_INTR_STARTED) {
 492                         for (index = 0; index < intr->n_alloc; index++)
 493                                 sfxge_ev_qmoderate(sc, index, moderation);
 494                 }
 495         } else {
 496                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 497                                    sizeof(sc->ev_moderation));
 498         }
 499
 500 out:
 501         sx_xunlock(&sc->softc_lock);
 502
 503         return error;
 504 }
 505
 506 static boolean_t
 507 sfxge_ev_initialized(void *arg)
 508 {
 509         struct sfxge_evq *evq;
 510
 511         evq = (struct sfxge_evq *)arg;
 512
 513         KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
 514             ("evq not starting"));
 515
 516         evq->init_state = SFXGE_EVQ_STARTED;
 517
 518         return (0);
 519 }
 520
 521 static boolean_t
 522 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 523 {
 524         struct sfxge_evq *evq;
 525         struct sfxge_softc *sc;
 526
 527         evq = (struct sfxge_evq *)arg;
 528         sc = evq->sc;
 529
 530         sfxge_mac_link_update(sc, link_mode);
 531
 532         return (0);
 533 }
 534
 535 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 536         .eec_initialized        = sfxge_ev_initialized,
 537         .eec_rx                 = sfxge_ev_rx,
 538         .eec_tx                 = sfxge_ev_tx,
 539         .eec_exception          = sfxge_ev_exception,
 540         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 541         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 542         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 543         .eec_software           = sfxge_ev_software,
 544         .eec_sram               = sfxge_ev_sram,
 545         .eec_wake_up            = sfxge_ev_wake_up,
 546         .eec_timer              = sfxge_ev_timer,
 547         .eec_link_change        = sfxge_ev_link_change,
 548 };
 549
 550
 551 int
 552 sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index)
 553 {
 554         struct sfxge_evq *evq;
 555         int rc;
 556
 557         evq = sc->evq[index];
 558
 559         mtx_lock(&evq->lock);
 560
 561         if (evq->init_state != SFXGE_EVQ_STARTING &&
 562             evq->init_state != SFXGE_EVQ_STARTED) {
 563                 rc = EINVAL;
 564                 goto fail;
 565         }
 566
 567         /* Synchronize the DMA memory for reading */
 568         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 569             BUS_DMASYNC_POSTREAD);
 570
 571         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 572         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 573         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 574         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 575
 576         /* Poll the queue */
 577         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 578
 579         evq->rx_done = 0;
 580         evq->tx_done = 0;
 581
 582         /* Perform any pending completion processing */
 583         sfxge_ev_qcomplete(evq, B_TRUE);
 584
 585         /* Re-prime the event queue for interrupts */
 586         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 587                 goto fail;
 588
 589         mtx_unlock(&evq->lock);
 590
 591         return (0);
 592
 593 fail:
 594         mtx_unlock(&(evq->lock));
 595         return (rc);
 596 }
 597
 598 static void
 599 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 600 {
 601         struct sfxge_evq *evq;
 602
 603         evq = sc->evq[index];
 604
 605         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 606             ("evq->init_state != SFXGE_EVQ_STARTED"));
 607
 608         mtx_lock(&evq->lock);
 609         evq->init_state = SFXGE_EVQ_INITIALIZED;
 610         evq->read_ptr = 0;
 611         evq->exception = B_FALSE;
 612
 613         /* Add event counts before discarding the common evq state */
 614         efx_ev_qstats_update(evq->common, sc->ev_stats);
 615
 616         efx_ev_qdestroy(evq->common);
 617         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 618             EFX_EVQ_NBUFS(SFXGE_NEVS));
 619         mtx_unlock(&evq->lock);
 620 }
 621
 622 static int
 623 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 624 {
 625         struct sfxge_evq *evq;
 626         efsys_mem_t *esmp;
 627         int count;
 628         int rc;
 629
 630         evq = sc->evq[index];
 631         esmp = &evq->mem;
 632
 633         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 634             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 635
 636         /* Clear all events. */
 637         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS));
 638
 639         /* Program the buffer table. */
 640         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 641             EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0)
 642                 return rc;
 643
 644         /* Create the common code event queue. */
 645         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS,
 646             evq->buf_base_id, &evq->common)) != 0)
 647                 goto fail;
 648
 649         mtx_lock(&evq->lock);
 650
 651         /* Set the default moderation */
 652         (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
 653
 654         /* Prime the event queue for interrupts */
 655         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 656                 goto fail2;
 657
 658         evq->init_state = SFXGE_EVQ_STARTING;
 659
 660         mtx_unlock(&evq->lock);
 661
 662         /* Wait for the initialization event */
 663         count = 0;
 664         do {
 665                 /* Pause for 100 ms */
 666                 pause("sfxge evq init", hz / 10);
 667
 668                 /* Check to see if the test event has been processed */
 669                 if (evq->init_state == SFXGE_EVQ_STARTED)
 670                         goto done;
 671
 672         } while (++count < 20);
 673
 674         rc = ETIMEDOUT;
 675         goto fail3;
 676
 677 done:
 678         return (0);
 679
 680 fail3:
 681         mtx_lock(&evq->lock);
 682         evq->init_state = SFXGE_EVQ_INITIALIZED;
 683 fail2:
 684         mtx_unlock(&evq->lock);
 685         efx_ev_qdestroy(evq->common);
 686 fail:
 687         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 688             EFX_EVQ_NBUFS(SFXGE_NEVS));
 689
 690         return (rc);
 691 }
 692
 693 void
 694 sfxge_ev_stop(struct sfxge_softc *sc)
 695 {
 696         struct sfxge_intr *intr;
 697         efx_nic_t *enp;
 698         int index;
 699
 700         intr = &sc->intr;
 701         enp = sc->enp;
 702
 703         KASSERT(intr->state == SFXGE_INTR_STARTED,
 704             ("Interrupts not started"));
 705
 706         /* Stop the event queue(s) */
 707         index = intr->n_alloc;
 708         while (--index >= 0)
 709                 sfxge_ev_qstop(sc, index);
 710
 711         /* Tear down the event module */
 712         efx_ev_fini(enp);
 713 }
 714
 715 int
 716 sfxge_ev_start(struct sfxge_softc *sc)
 717 {
 718         struct sfxge_intr *intr;
 719         int index;
 720         int rc;
 721
 722         intr = &sc->intr;
 723
 724         KASSERT(intr->state == SFXGE_INTR_STARTED,
 725             ("intr->state != SFXGE_INTR_STARTED"));
 726
 727         /* Initialize the event module */
 728         if ((rc = efx_ev_init(sc->enp)) != 0)
 729                 return rc;
 730
 731         /* Start the event queues */
 732         for (index = 0; index < intr->n_alloc; index++) {
 733                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 734                         goto fail;
 735         }
 736
 737         return (0);
 738
 739 fail:
 740         /* Stop the event queue(s) */
 741         while (--index >= 0)
 742                 sfxge_ev_qstop(sc, index);
 743
 744         /* Tear down the event module */
 745         efx_ev_fini(sc->enp);
 746
 747         return (rc);
 748 }
 749
 750 static void
 751 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 752 {
 753         struct sfxge_evq *evq;
 754
 755         evq = sc->evq[index];
 756
 757         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 758             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 759         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 760
 761         sfxge_dma_free(&evq->mem);
 762
 763         sc->evq[index] = NULL;
 764
 765         mtx_destroy(&evq->lock);
 766
 767         free(evq, M_SFXGE);
 768 }
 769
 770 static int
 771 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 772 {
 773         struct sfxge_evq *evq;
 774         efsys_mem_t *esmp;
 775         int rc;
 776
 777         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 778
 779         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 780         evq->sc = sc;
 781         evq->index = index;
 782         sc->evq[index] = evq;
 783         esmp = &evq->mem;
 784
 785         /* Initialise TX completion list */
 786         evq->txqs = &evq->txq;
 787
 788         /* Allocate DMA space. */
 789         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0)
 790                 return (rc);
 791
 792         /* Allocate buffer table entries. */
 793         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS),
 794                                  &evq->buf_base_id);
 795
 796         mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
 797
 798         evq->init_state = SFXGE_EVQ_INITIALIZED;
 799
 800         return (0);
 801 }
 802
 803 void
 804 sfxge_ev_fini(struct sfxge_softc *sc)
 805 {
 806         struct sfxge_intr *intr;
 807         int index;
 808
 809         intr = &sc->intr;
 810
 811         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 812             ("intr->state != SFXGE_INTR_INITIALIZED"));
 813
 814         sc->ev_moderation = 0;
 815
 816         /* Tear down the event queue(s). */
 817         index = intr->n_alloc;
 818         while (--index >= 0)
 819                 sfxge_ev_qfini(sc, index);
 820 }
 821
 822 int
 823 sfxge_ev_init(struct sfxge_softc *sc)
 824 {
 825         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 826         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 827         struct sfxge_intr *intr;
 828         int index;
 829         int rc;
 830
 831         intr = &sc->intr;
 832
 833         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 834             ("intr->state != SFXGE_INTR_INITIALIZED"));
 835
 836         /* Set default interrupt moderation; add a sysctl to
 837          * read and change it.
 838          */
 839         sc->ev_moderation = 30;
 840         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 841                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 842                         sc, 0, sfxge_int_mod_handler, "IU",
 843                         "sfxge interrupt moderation (us)");
 844
 845         /*
 846          * Initialize the event queue(s) - one per interrupt.
 847          */
 848         for (index = 0; index < intr->n_alloc; index++) {
 849                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 850                         goto fail;
 851         }
 852
 853         sfxge_ev_stat_init(sc);
 854
 855         return (0);
 856
 857 fail:
 858         while (--index >= 0)
 859                 sfxge_ev_qfini(sc, index);
 860
 861         return (rc);
 862 }