sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/kernel.h>
  35 #include <sys/malloc.h>
  36 #include <sys/param.h>
  37 #include <sys/queue.h>
  38 #include <sys/systm.h>
  39 #include <sys/taskqueue.h>
  40
  41 #include "common/efx.h"
  42
  43 #include "sfxge.h"
  44
  45 static void
  46 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  47 {
  48         struct sfxge_softc *sc;
  49         unsigned int index;
  50         struct sfxge_rxq *rxq;
  51         struct sfxge_txq *txq;
  52
  53         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  54
  55         sc = evq->sc;
  56         index = evq->index;
  57         rxq = sc->rxq[index];
  58
  59         if ((txq = evq->txq) != NULL) {
  60                 evq->txq = NULL;
  61                 evq->txqs = &(evq->txq);
  62
  63                 do {
  64                         struct sfxge_txq *next;
  65
  66                         next = txq->next;
  67                         txq->next = NULL;
  68
  69                         KASSERT(txq->evq_index == index,
  70                             ("txq->evq_index != index"));
  71
  72                         if (txq->pending != txq->completed)
  73                                 sfxge_tx_qcomplete(txq, evq);
  74
  75                         txq = next;
  76                 } while (txq != NULL);
  77         }
  78
  79         if (rxq->pending != rxq->completed)
  80                 sfxge_rx_qcomplete(rxq, eop);
  81 }
  82
  83 static boolean_t
  84 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  85     uint16_t flags)
  86 {
  87         struct sfxge_evq *evq;
  88         struct sfxge_softc *sc;
  89         struct sfxge_rxq *rxq;
  90         unsigned int expected;
  91         struct sfxge_rx_sw_desc *rx_desc;
  92
  93         evq = arg;
  94         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  95
  96         sc = evq->sc;
  97
  98         if (evq->exception)
  99                 goto done;
 100
 101         rxq = sc->rxq[label];
 102         KASSERT(rxq != NULL, ("rxq == NULL"));
 103         KASSERT(evq->index == rxq->index,
 104             ("evq->index != rxq->index"));
 105
 106         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
 107                 goto done;
 108
 109         expected = rxq->pending++ & rxq->ptr_mask;
 110         if (id != expected) {
 111                 evq->exception = B_TRUE;
 112
 113                 device_printf(sc->dev, "RX completion out of order"
 114                               " (id=%#x expected=%#x flags=%#x); resetting\n",
 115                               id, expected, flags);
 116                 sfxge_schedule_reset(sc);
 117
 118                 goto done;
 119         }
 120
 121         rx_desc = &rxq->queue[id];
 122
 123         KASSERT(rx_desc->flags == EFX_DISCARD,
 124             ("rx_desc->flags != EFX_DISCARD"));
 125         rx_desc->flags = flags;
 126
 127         KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 128         rx_desc->size = (uint16_t)size;
 129         prefetch_read_many(rx_desc->mbuf);
 130
 131         evq->rx_done++;
 132
 133         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 134                 sfxge_ev_qcomplete(evq, B_FALSE);
 135
 136 done:
 137         return (evq->rx_done >= SFXGE_EV_BATCH);
 138 }
 139
 140 static boolean_t
 141 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 142 {
 143         struct sfxge_evq *evq;
 144         struct sfxge_softc *sc;
 145
 146         evq = (struct sfxge_evq *)arg;
 147         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 148
 149         sc = evq->sc;
 150
 151         evq->exception = B_TRUE;
 152
 153         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 154                 device_printf(sc->dev,
 155                               "hardware exception (code=%u); resetting\n",
 156                               code);
 157                 sfxge_schedule_reset(sc);
 158         }
 159
 160         return (B_FALSE);
 161 }
 162
 163 static boolean_t
 164 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
 165 {
 166         struct sfxge_evq *evq;
 167         struct sfxge_softc *sc;
 168         struct sfxge_rxq *rxq;
 169         unsigned int index;
 170         unsigned int label;
 171         uint16_t magic;
 172
 173         evq = (struct sfxge_evq *)arg;
 174         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 175
 176         sc = evq->sc;
 177         rxq = sc->rxq[rxq_index];
 178
 179         KASSERT(rxq != NULL, ("rxq == NULL"));
 180
 181         /* Resend a software event on the correct queue */
 182         index = rxq->index;
 183         evq = sc->evq[index];
 184
 185         label = rxq_index;
 186         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 187             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
 188         magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
 189
 190         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 191             ("evq not started"));
 192         efx_ev_qpost(evq->common, magic);
 193
 194         return (B_FALSE);
 195 }
 196
 197 static boolean_t
 198 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
 199 {
 200         struct sfxge_evq *evq;
 201         struct sfxge_softc *sc;
 202         struct sfxge_rxq *rxq;
 203         unsigned int index;
 204         unsigned int label;
 205         uint16_t magic;
 206
 207         evq = (struct sfxge_evq *)arg;
 208         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 209
 210         sc = evq->sc;
 211         rxq = sc->rxq[rxq_index];
 212
 213         KASSERT(rxq != NULL, ("rxq == NULL"));
 214
 215         /* Resend a software event on the correct queue */
 216         index = rxq->index;
 217         evq = sc->evq[index];
 218
 219         label = rxq_index;
 220         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 221             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 222         magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
 223
 224         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 225             ("evq not started"));
 226         efx_ev_qpost(evq->common, magic);
 227
 228         return (B_FALSE);
 229 }
 230
 231 static struct sfxge_txq *
 232 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
 233 {
 234         unsigned int index;
 235
 236         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
 237             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
 238         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
 239         return (evq->sc->txq[index]);
 240 }
 241
 242 static boolean_t
 243 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 244 {
 245         struct sfxge_evq *evq;
 246         struct sfxge_txq *txq;
 247         unsigned int stop;
 248         unsigned int delta;
 249
 250         evq = (struct sfxge_evq *)arg;
 251         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 252
 253         txq = sfxge_get_txq_by_label(evq, label);
 254
 255         KASSERT(txq != NULL, ("txq == NULL"));
 256         KASSERT(evq->index == txq->evq_index,
 257             ("evq->index != txq->evq_index"));
 258
 259         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
 260                 goto done;
 261
 262         stop = (id + 1) & txq->ptr_mask;
 263         id = txq->pending & txq->ptr_mask;
 264
 265         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
 266         txq->pending += delta;
 267
 268         evq->tx_done++;
 269
 270         if (txq->next == NULL &&
 271             evq->txqs != &(txq->next)) {
 272                 *(evq->txqs) = txq;
 273                 evq->txqs = &(txq->next);
 274         }
 275
 276         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 277                 sfxge_tx_qcomplete(txq, evq);
 278
 279 done:
 280         return (evq->tx_done >= SFXGE_EV_BATCH);
 281 }
 282
 283 static boolean_t
 284 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
 285 {
 286         struct sfxge_evq *evq;
 287         struct sfxge_softc *sc;
 288         struct sfxge_txq *txq;
 289         unsigned int label;
 290         uint16_t magic;
 291
 292         evq = (struct sfxge_evq *)arg;
 293         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 294
 295         sc = evq->sc;
 296         txq = sc->txq[txq_index];
 297
 298         KASSERT(txq != NULL, ("txq == NULL"));
 299         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 300             ("txq not initialized"));
 301
 302         /* Resend a software event on the correct queue */
 303         evq = sc->evq[txq->evq_index];
 304
 305         label = txq->type;
 306         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 307             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 308         magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
 309
 310         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 311             ("evq not started"));
 312         efx_ev_qpost(evq->common, magic);
 313
 314         return (B_FALSE);
 315 }
 316
 317 static boolean_t
 318 sfxge_ev_software(void *arg, uint16_t magic)
 319 {
 320         struct sfxge_evq *evq;
 321         struct sfxge_softc *sc;
 322         unsigned int label;
 323
 324         evq = (struct sfxge_evq *)arg;
 325         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 326
 327         sc = evq->sc;
 328
 329         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 330         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 331
 332         switch (magic) {
 333         case SFXGE_MAGIC_RX_QFLUSH_DONE: {
 334                 struct sfxge_rxq *rxq = sc->rxq[label];
 335
 336                 KASSERT(rxq != NULL, ("rxq == NULL"));
 337                 KASSERT(evq->index == rxq->index,
 338                     ("evq->index != rxq->index"));
 339
 340                 sfxge_rx_qflush_done(rxq);
 341                 break;
 342         }
 343         case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
 344                 struct sfxge_rxq *rxq = sc->rxq[label];
 345
 346                 KASSERT(rxq != NULL, ("rxq == NULL"));
 347                 KASSERT(evq->index == rxq->index,
 348                     ("evq->index != rxq->index"));
 349
 350                 sfxge_rx_qflush_failed(rxq);
 351                 break;
 352         }
 353         case SFXGE_MAGIC_RX_QREFILL: {
 354                 struct sfxge_rxq *rxq = sc->rxq[label];
 355
 356                 KASSERT(rxq != NULL, ("rxq == NULL"));
 357                 KASSERT(evq->index == rxq->index,
 358                     ("evq->index != rxq->index"));
 359
 360                 sfxge_rx_qrefill(rxq);
 361                 break;
 362         }
 363         case SFXGE_MAGIC_TX_QFLUSH_DONE: {
 364                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
 365
 366                 KASSERT(txq != NULL, ("txq == NULL"));
 367                 KASSERT(evq->index == txq->evq_index,
 368                     ("evq->index != txq->evq_index"));
 369
 370                 sfxge_tx_qflush_done(txq);
 371                 break;
 372         }
 373         default:
 374                 break;
 375         }
 376
 377         return (B_FALSE);
 378 }
 379
 380 static boolean_t
 381 sfxge_ev_sram(void *arg, uint32_t code)
 382 {
 383         (void)arg;
 384         (void)code;
 385
 386         switch (code) {
 387         case EFX_SRAM_UPDATE:
 388                 EFSYS_PROBE(sram_update);
 389                 break;
 390
 391         case EFX_SRAM_CLEAR:
 392                 EFSYS_PROBE(sram_clear);
 393                 break;
 394
 395         case EFX_SRAM_ILLEGAL_CLEAR:
 396                 EFSYS_PROBE(sram_illegal_clear);
 397                 break;
 398
 399         default:
 400                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 401                 break;
 402         }
 403
 404         return (B_FALSE);
 405 }
 406
 407 static boolean_t
 408 sfxge_ev_timer(void *arg, uint32_t index)
 409 {
 410         (void)arg;
 411         (void)index;
 412
 413         return (B_FALSE);
 414 }
 415
 416 static boolean_t
 417 sfxge_ev_wake_up(void *arg, uint32_t index)
 418 {
 419         (void)arg;
 420         (void)index;
 421
 422         return (B_FALSE);
 423 }
 424
 425 #if EFSYS_OPT_QSTATS
 426
 427 static void
 428 sfxge_ev_stat_update(struct sfxge_softc *sc)
 429 {
 430         struct sfxge_evq *evq;
 431         unsigned int index;
 432         clock_t now;
 433
 434         SFXGE_ADAPTER_LOCK(sc);
 435
 436         if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
 437                 goto out;
 438
 439         now = ticks;
 440         if (now - sc->ev_stats_update_time < hz)
 441                 goto out;
 442
 443         sc->ev_stats_update_time = now;
 444
 445         /* Add event counts from each event queue in turn */
 446         for (index = 0; index < sc->evq_count; index++) {
 447                 evq = sc->evq[index];
 448                 SFXGE_EVQ_LOCK(evq);
 449                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 450                 SFXGE_EVQ_UNLOCK(evq);
 451         }
 452 out:
 453         SFXGE_ADAPTER_UNLOCK(sc);
 454 }
 455
 456 static int
 457 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 458 {
 459         struct sfxge_softc *sc = arg1;
 460         unsigned int id = arg2;
 461
 462         sfxge_ev_stat_update(sc);
 463
 464         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
 465 }
 466
 467 static void
 468 sfxge_ev_stat_init(struct sfxge_softc *sc)
 469 {
 470         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 471         struct sysctl_oid_list *stat_list;
 472         unsigned int id;
 473         char name[40];
 474
 475         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 476
 477         for (id = 0; id < EV_NQSTATS; id++) {
 478                 snprintf(name, sizeof(name), "ev_%s",
 479                          efx_ev_qstat_name(sc->enp, id));
 480                 SYSCTL_ADD_PROC(
 481                         ctx, stat_list,
 482                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 483                         sc, id, sfxge_ev_stat_handler, "Q",
 484                         "");
 485         }
 486 }
 487
 488 #endif /* EFSYS_OPT_QSTATS */
 489
 490 static void
 491 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 492 {
 493         struct sfxge_evq *evq;
 494         efx_evq_t *eep;
 495
 496         evq = sc->evq[idx];
 497         eep = evq->common;
 498
 499         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 500             ("evq->init_state != SFXGE_EVQ_STARTED"));
 501
 502         (void)efx_ev_qmoderate(eep, us);
 503 }
 504
 505 static int
 506 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 507 {
 508         struct sfxge_softc *sc = arg1;
 509         struct sfxge_intr *intr = &sc->intr;
 510         unsigned int moderation;
 511         int error;
 512         unsigned int index;
 513
 514         SFXGE_ADAPTER_LOCK(sc);
 515
 516         if (req->newptr != NULL) {
 517                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 518                     != 0)
 519                         goto out;
 520
 521                 /* We may not be calling efx_ev_qmoderate() now,
 522                  * so we have to range-check the value ourselves.
 523                  */
 524                 if (moderation >
 525                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
 526                         error = EINVAL;
 527                         goto out;
 528                 }
 529
 530                 sc->ev_moderation = moderation;
 531                 if (intr->state == SFXGE_INTR_STARTED) {
 532                         for (index = 0; index < sc->evq_count; index++)
 533                                 sfxge_ev_qmoderate(sc, index, moderation);
 534                 }
 535         } else {
 536                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 537                                    sizeof(sc->ev_moderation));
 538         }
 539
 540 out:
 541         SFXGE_ADAPTER_UNLOCK(sc);
 542
 543         return (error);
 544 }
 545
 546 static boolean_t
 547 sfxge_ev_initialized(void *arg)
 548 {
 549         struct sfxge_evq *evq;
 550
 551         evq = (struct sfxge_evq *)arg;
 552         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 553
 554         KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
 555             ("evq not starting"));
 556
 557         evq->init_state = SFXGE_EVQ_STARTED;
 558
 559         return (0);
 560 }
 561
 562 static boolean_t
 563 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 564 {
 565         struct sfxge_evq *evq;
 566         struct sfxge_softc *sc;
 567
 568         evq = (struct sfxge_evq *)arg;
 569         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 570
 571         sc = evq->sc;
 572
 573         sfxge_mac_link_update(sc, link_mode);
 574
 575         return (0);
 576 }
 577
 578 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 579         .eec_initialized        = sfxge_ev_initialized,
 580         .eec_rx                 = sfxge_ev_rx,
 581         .eec_tx                 = sfxge_ev_tx,
 582         .eec_exception          = sfxge_ev_exception,
 583         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 584         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 585         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 586         .eec_software           = sfxge_ev_software,
 587         .eec_sram               = sfxge_ev_sram,
 588         .eec_wake_up            = sfxge_ev_wake_up,
 589         .eec_timer              = sfxge_ev_timer,
 590         .eec_link_change        = sfxge_ev_link_change,
 591 };
 592
 593
 594 int
 595 sfxge_ev_qpoll(struct sfxge_evq *evq)
 596 {
 597         int rc;
 598
 599         SFXGE_EVQ_LOCK(evq);
 600
 601         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
 602                             evq->init_state != SFXGE_EVQ_STARTED)) {
 603                 rc = EINVAL;
 604                 goto fail;
 605         }
 606
 607         /* Synchronize the DMA memory for reading */
 608         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 609             BUS_DMASYNC_POSTREAD);
 610
 611         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 612         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 613         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 614         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 615
 616         /* Poll the queue */
 617         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 618
 619         evq->rx_done = 0;
 620         evq->tx_done = 0;
 621
 622         /* Perform any pending completion processing */
 623         sfxge_ev_qcomplete(evq, B_TRUE);
 624
 625         /* Re-prime the event queue for interrupts */
 626         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 627                 goto fail;
 628
 629         SFXGE_EVQ_UNLOCK(evq);
 630
 631         return (0);
 632
 633 fail:
 634         SFXGE_EVQ_UNLOCK(evq);
 635         return (rc);
 636 }
 637
 638 static void
 639 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 640 {
 641         struct sfxge_evq *evq;
 642
 643         evq = sc->evq[index];
 644
 645         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 646             ("evq->init_state != SFXGE_EVQ_STARTED"));
 647
 648         SFXGE_EVQ_LOCK(evq);
 649         evq->init_state = SFXGE_EVQ_INITIALIZED;
 650         evq->read_ptr = 0;
 651         evq->exception = B_FALSE;
 652
 653 #if EFSYS_OPT_QSTATS
 654         /* Add event counts before discarding the common evq state */
 655         efx_ev_qstats_update(evq->common, sc->ev_stats);
 656 #endif
 657
 658         efx_ev_qdestroy(evq->common);
 659         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 660             EFX_EVQ_NBUFS(evq->entries));
 661         SFXGE_EVQ_UNLOCK(evq);
 662 }
 663
 664 static int
 665 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 666 {
 667         struct sfxge_evq *evq;
 668         efsys_mem_t *esmp;
 669         int count;
 670         int rc;
 671
 672         evq = sc->evq[index];
 673         esmp = &evq->mem;
 674
 675         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 676             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 677
 678         /* Clear all events. */
 679         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
 680
 681         /* Program the buffer table. */
 682         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 683             EFX_EVQ_NBUFS(evq->entries))) != 0)
 684                 return (rc);
 685
 686         /* Create the common code event queue. */
 687         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
 688             evq->buf_base_id, &evq->common)) != 0)
 689                 goto fail;
 690
 691         SFXGE_EVQ_LOCK(evq);
 692
 693         /* Set the default moderation */
 694         (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
 695
 696         /* Prime the event queue for interrupts */
 697         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 698                 goto fail2;
 699
 700         evq->init_state = SFXGE_EVQ_STARTING;
 701
 702         SFXGE_EVQ_UNLOCK(evq);
 703
 704         /* Wait for the initialization event */
 705         count = 0;
 706         do {
 707                 /* Pause for 100 ms */
 708                 pause("sfxge evq init", hz / 10);
 709
 710                 /* Check to see if the test event has been processed */
 711                 if (evq->init_state == SFXGE_EVQ_STARTED)
 712                         goto done;
 713
 714         } while (++count < 20);
 715
 716         rc = ETIMEDOUT;
 717         goto fail3;
 718
 719 done:
 720         return (0);
 721
 722 fail3:
 723         SFXGE_EVQ_LOCK(evq);
 724         evq->init_state = SFXGE_EVQ_INITIALIZED;
 725 fail2:
 726         SFXGE_EVQ_UNLOCK(evq);
 727         efx_ev_qdestroy(evq->common);
 728 fail:
 729         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 730             EFX_EVQ_NBUFS(evq->entries));
 731
 732         return (rc);
 733 }
 734
 735 void
 736 sfxge_ev_stop(struct sfxge_softc *sc)
 737 {
 738         struct sfxge_intr *intr;
 739         efx_nic_t *enp;
 740         int index;
 741
 742         intr = &sc->intr;
 743         enp = sc->enp;
 744
 745         KASSERT(intr->state == SFXGE_INTR_STARTED,
 746             ("Interrupts not started"));
 747
 748         /* Stop the event queue(s) */
 749         index = sc->evq_count;
 750         while (--index >= 0)
 751                 sfxge_ev_qstop(sc, index);
 752
 753         /* Tear down the event module */
 754         efx_ev_fini(enp);
 755 }
 756
 757 int
 758 sfxge_ev_start(struct sfxge_softc *sc)
 759 {
 760         struct sfxge_intr *intr;
 761         int index;
 762         int rc;
 763
 764         intr = &sc->intr;
 765
 766         KASSERT(intr->state == SFXGE_INTR_STARTED,
 767             ("intr->state != SFXGE_INTR_STARTED"));
 768
 769         /* Initialize the event module */
 770         if ((rc = efx_ev_init(sc->enp)) != 0)
 771                 return (rc);
 772
 773         /* Start the event queues */
 774         for (index = 0; index < sc->evq_count; index++) {
 775                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 776                         goto fail;
 777         }
 778
 779         return (0);
 780
 781 fail:
 782         /* Stop the event queue(s) */
 783         while (--index >= 0)
 784                 sfxge_ev_qstop(sc, index);
 785
 786         /* Tear down the event module */
 787         efx_ev_fini(sc->enp);
 788
 789         return (rc);
 790 }
 791
 792 static void
 793 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 794 {
 795         struct sfxge_evq *evq;
 796
 797         evq = sc->evq[index];
 798
 799         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 800             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 801         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 802
 803         sfxge_dma_free(&evq->mem);
 804
 805         sc->evq[index] = NULL;
 806
 807         SFXGE_EVQ_LOCK_DESTROY(evq);
 808
 809         free(evq, M_SFXGE);
 810 }
 811
 812 static int
 813 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 814 {
 815         struct sfxge_evq *evq;
 816         efsys_mem_t *esmp;
 817         int rc;
 818
 819         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 820
 821         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 822         evq->sc = sc;
 823         evq->index = index;
 824         sc->evq[index] = evq;
 825         esmp = &evq->mem;
 826
 827         /* Build an event queue with room for one event per tx and rx buffer,
 828          * plus some extra for link state events and MCDI completions.
 829          * There are three tx queues in the first event queue and one in
 830          * other.
 831          */
 832         if (index == 0)
 833                 evq->entries =
 834                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 835                                            3 * sc->txq_entries +
 836                                            128);
 837         else
 838                 evq->entries =
 839                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 840                                            sc->txq_entries +
 841                                            128);
 842
 843         /* Initialise TX completion list */
 844         evq->txqs = &evq->txq;
 845
 846         /* Allocate DMA space. */
 847         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
 848                 return (rc);
 849
 850         /* Allocate buffer table entries. */
 851         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
 852                                  &evq->buf_base_id);
 853
 854         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
 855
 856         evq->init_state = SFXGE_EVQ_INITIALIZED;
 857
 858         return (0);
 859 }
 860
 861 void
 862 sfxge_ev_fini(struct sfxge_softc *sc)
 863 {
 864         struct sfxge_intr *intr;
 865         int index;
 866
 867         intr = &sc->intr;
 868
 869         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 870             ("intr->state != SFXGE_INTR_INITIALIZED"));
 871
 872         sc->ev_moderation = 0;
 873
 874         /* Tear down the event queue(s). */
 875         index = sc->evq_count;
 876         while (--index >= 0)
 877                 sfxge_ev_qfini(sc, index);
 878
 879         sc->evq_count = 0;
 880 }
 881
 882 int
 883 sfxge_ev_init(struct sfxge_softc *sc)
 884 {
 885         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 886         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 887         struct sfxge_intr *intr;
 888         int index;
 889         int rc;
 890
 891         intr = &sc->intr;
 892
 893         sc->evq_count = intr->n_alloc;
 894
 895         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 896             ("intr->state != SFXGE_INTR_INITIALIZED"));
 897
 898         /* Set default interrupt moderation; add a sysctl to
 899          * read and change it.
 900          */
 901         sc->ev_moderation = SFXGE_MODERATION;
 902         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 903                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 904                         sc, 0, sfxge_int_mod_handler, "IU",
 905                         "sfxge interrupt moderation (us)");
 906
 907         /*
 908          * Initialize the event queue(s) - one per interrupt.
 909          */
 910         for (index = 0; index < sc->evq_count; index++) {
 911                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 912                         goto fail;
 913         }
 914
 915 #if EFSYS_OPT_QSTATS
 916         sfxge_ev_stat_init(sc);
 917 #endif
 918
 919         return (0);
 920
 921 fail:
 922         while (--index >= 0)
 923                 sfxge_ev_qfini(sc, index);
 924
 925         sc->evq_count = 0;
 926         return (rc);
 927 }