sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright notice,
  12  *    this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  *    this list of conditions and the following disclaimer in the documentation
  15  *    and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * The views and conclusions contained in the software and documentation are
  30  * those of the authors and should not be interpreted as representing official
  31  * policies, either expressed or implied, of the FreeBSD Project.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39
  40 #include "common/efx.h"
  41
  42 #include "sfxge.h"
  43
  44 static void
  45 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  46 {
  47         struct sfxge_softc *sc;
  48         unsigned int index;
  49         struct sfxge_rxq *rxq;
  50         struct sfxge_txq *txq;
  51
  52         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  53
  54         sc = evq->sc;
  55         index = evq->index;
  56         rxq = sc->rxq[index];
  57
  58         if ((txq = evq->txq) != NULL) {
  59                 evq->txq = NULL;
  60                 evq->txqs = &(evq->txq);
  61
  62                 do {
  63                         struct sfxge_txq *next;
  64
  65                         next = txq->next;
  66                         txq->next = NULL;
  67
  68                         KASSERT(txq->evq_index == index,
  69                             ("txq->evq_index != index"));
  70
  71                         if (txq->pending != txq->completed)
  72                                 sfxge_tx_qcomplete(txq, evq);
  73
  74                         txq = next;
  75                 } while (txq != NULL);
  76         }
  77
  78         if (rxq->pending != rxq->completed)
  79                 sfxge_rx_qcomplete(rxq, eop);
  80 }
  81
  82 static struct sfxge_rxq *
  83 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
  84 {
  85         struct sfxge_rxq *rxq;
  86
  87         KASSERT(label == 0, ("unexpected rxq label != 0"));
  88
  89         rxq = evq->sc->rxq[evq->index];
  90
  91         KASSERT(rxq != NULL, ("rxq == NULL"));
  92         KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
  93
  94         return (rxq);
  95 }
  96
  97 static boolean_t
  98 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  99             uint16_t flags)
 100 {
 101         struct sfxge_evq *evq;
 102         struct sfxge_softc *sc;
 103         struct sfxge_rxq *rxq;
 104         unsigned int stop;
 105         unsigned int delta;
 106         struct sfxge_rx_sw_desc *rx_desc;
 107
 108         evq = arg;
 109         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 110
 111         sc = evq->sc;
 112
 113         if (evq->exception)
 114                 goto done;
 115
 116         rxq = sfxge_get_rxq_by_label(evq, label);
 117         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
 118                 goto done;
 119
 120         stop = (id + 1) & rxq->ptr_mask;
 121         id = rxq->pending & rxq->ptr_mask;
 122         delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
 123         rxq->pending += delta;
 124
 125         if (delta != 1) {
 126                 if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) ||
 127                     (delta <= 0) ||
 128                     (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
 129                         evq->exception = B_TRUE;
 130
 131                         device_printf(sc->dev, "RX completion out of order"
 132                                                   " (id=%#x delta=%u flags=%#x); resetting\n",
 133                                                   id, delta, flags);
 134                         sfxge_schedule_reset(sc);
 135
 136                         goto done;
 137                 }
 138         }
 139
 140         rx_desc = &rxq->queue[id];
 141
 142         prefetch_read_many(rx_desc->mbuf);
 143
 144         for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
 145                 rx_desc = &rxq->queue[id];
 146                 KASSERT(rx_desc->flags == EFX_DISCARD,
 147                                 ("rx_desc->flags != EFX_DISCARD"));
 148                 rx_desc->flags = flags;
 149
 150                 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 151                 rx_desc->size = (uint16_t)size;
 152         }
 153
 154         evq->rx_done++;
 155
 156         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 157                 sfxge_ev_qcomplete(evq, B_FALSE);
 158
 159 done:
 160         return (evq->rx_done >= SFXGE_EV_BATCH);
 161 }
 162
 163 static boolean_t
 164 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 165 {
 166         struct sfxge_evq *evq;
 167         struct sfxge_softc *sc;
 168
 169         evq = (struct sfxge_evq *)arg;
 170         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 171
 172         sc = evq->sc;
 173
 174         DBGPRINT(sc->dev, "[%d] %s", evq->index,
 175                           (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
 176                           (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
 177                           (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
 178                           (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
 179                           (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
 180                           (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
 181                           (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
 182                           (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
 183                           (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
 184                           "UNKNOWN");
 185
 186         evq->exception = B_TRUE;
 187
 188         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 189                 device_printf(sc->dev,
 190                               "hardware exception (code=%u); resetting\n",
 191                               code);
 192                 sfxge_schedule_reset(sc);
 193         }
 194
 195         return (B_FALSE);
 196 }
 197
 198 static boolean_t
 199 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
 200 {
 201         struct sfxge_evq *evq;
 202         struct sfxge_softc *sc;
 203         struct sfxge_rxq *rxq;
 204         unsigned int index;
 205         uint16_t magic;
 206
 207         evq = (struct sfxge_evq *)arg;
 208         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 209
 210         sc = evq->sc;
 211         rxq = sc->rxq[rxq_index];
 212
 213         KASSERT(rxq != NULL, ("rxq == NULL"));
 214
 215         /* Resend a software event on the correct queue */
 216         index = rxq->index;
 217         if (index == evq->index) {
 218                 sfxge_rx_qflush_done(rxq);
 219                 return (B_FALSE);
 220         }
 221
 222         evq = sc->evq[index];
 223         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
 224
 225         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 226             ("evq not started"));
 227         efx_ev_qpost(evq->common, magic);
 228
 229         return (B_FALSE);
 230 }
 231
 232 static boolean_t
 233 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
 234 {
 235         struct sfxge_evq *evq;
 236         struct sfxge_softc *sc;
 237         struct sfxge_rxq *rxq;
 238         unsigned int index;
 239         uint16_t magic;
 240
 241         evq = (struct sfxge_evq *)arg;
 242         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 243
 244         sc = evq->sc;
 245         rxq = sc->rxq[rxq_index];
 246
 247         KASSERT(rxq != NULL, ("rxq == NULL"));
 248
 249         /* Resend a software event on the correct queue */
 250         index = rxq->index;
 251         evq = sc->evq[index];
 252         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
 253
 254         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 255             ("evq not started"));
 256         efx_ev_qpost(evq->common, magic);
 257
 258         return (B_FALSE);
 259 }
 260
 261 static struct sfxge_txq *
 262 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
 263 {
 264         unsigned int index;
 265
 266         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
 267             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
 268         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
 269         return (evq->sc->txq[index]);
 270 }
 271
 272 static boolean_t
 273 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 274 {
 275         struct sfxge_evq *evq;
 276         struct sfxge_txq *txq;
 277         unsigned int stop;
 278         unsigned int delta;
 279
 280         evq = (struct sfxge_evq *)arg;
 281         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 282
 283         txq = sfxge_get_txq_by_label(evq, label);
 284
 285         KASSERT(txq != NULL, ("txq == NULL"));
 286         KASSERT(evq->index == txq->evq_index,
 287             ("evq->index != txq->evq_index"));
 288
 289         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
 290                 goto done;
 291
 292         stop = (id + 1) & txq->ptr_mask;
 293         id = txq->pending & txq->ptr_mask;
 294
 295         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
 296         txq->pending += delta;
 297
 298         evq->tx_done++;
 299
 300         if (txq->next == NULL &&
 301             evq->txqs != &(txq->next)) {
 302                 *(evq->txqs) = txq;
 303                 evq->txqs = &(txq->next);
 304         }
 305
 306         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 307                 sfxge_tx_qcomplete(txq, evq);
 308
 309 done:
 310         return (evq->tx_done >= SFXGE_EV_BATCH);
 311 }
 312
 313 static boolean_t
 314 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
 315 {
 316         struct sfxge_evq *evq;
 317         struct sfxge_softc *sc;
 318         struct sfxge_txq *txq;
 319         uint16_t magic;
 320
 321         evq = (struct sfxge_evq *)arg;
 322         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 323
 324         sc = evq->sc;
 325         txq = sc->txq[txq_index];
 326
 327         KASSERT(txq != NULL, ("txq == NULL"));
 328         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 329             ("txq not initialized"));
 330
 331         if (txq->evq_index == evq->index) {
 332                 sfxge_tx_qflush_done(txq);
 333                 return (B_FALSE);
 334         }
 335
 336         /* Resend a software event on the correct queue */
 337         evq = sc->evq[txq->evq_index];
 338         magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
 339
 340         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 341             ("evq not started"));
 342         efx_ev_qpost(evq->common, magic);
 343
 344         return (B_FALSE);
 345 }
 346
 347 static boolean_t
 348 sfxge_ev_software(void *arg, uint16_t magic)
 349 {
 350         struct sfxge_evq *evq;
 351         struct sfxge_softc *sc;
 352         unsigned int label;
 353
 354         evq = (struct sfxge_evq *)arg;
 355         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 356
 357         sc = evq->sc;
 358
 359         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 360         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 361
 362         switch (magic) {
 363         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
 364                 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
 365                 break;
 366
 367         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
 368                 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
 369                 break;
 370
 371         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
 372                 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
 373                 break;
 374
 375         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
 376                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
 377
 378                 KASSERT(txq != NULL, ("txq == NULL"));
 379                 KASSERT(evq->index == txq->evq_index,
 380                     ("evq->index != txq->evq_index"));
 381
 382                 sfxge_tx_qflush_done(txq);
 383                 break;
 384         }
 385         default:
 386                 break;
 387         }
 388
 389         return (B_FALSE);
 390 }
 391
 392 static boolean_t
 393 sfxge_ev_sram(void *arg, uint32_t code)
 394 {
 395         (void)arg;
 396         (void)code;
 397
 398         switch (code) {
 399         case EFX_SRAM_UPDATE:
 400                 EFSYS_PROBE(sram_update);
 401                 break;
 402
 403         case EFX_SRAM_CLEAR:
 404                 EFSYS_PROBE(sram_clear);
 405                 break;
 406
 407         case EFX_SRAM_ILLEGAL_CLEAR:
 408                 EFSYS_PROBE(sram_illegal_clear);
 409                 break;
 410
 411         default:
 412                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 413                 break;
 414         }
 415
 416         return (B_FALSE);
 417 }
 418
 419 static boolean_t
 420 sfxge_ev_timer(void *arg, uint32_t index)
 421 {
 422         (void)arg;
 423         (void)index;
 424
 425         return (B_FALSE);
 426 }
 427
 428 static boolean_t
 429 sfxge_ev_wake_up(void *arg, uint32_t index)
 430 {
 431         (void)arg;
 432         (void)index;
 433
 434         return (B_FALSE);
 435 }
 436
 437 #if EFSYS_OPT_QSTATS
 438
 439 static void
 440 sfxge_ev_stat_update(struct sfxge_softc *sc)
 441 {
 442         struct sfxge_evq *evq;
 443         unsigned int index;
 444         clock_t now;
 445
 446         SFXGE_ADAPTER_LOCK(sc);
 447
 448         if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
 449                 goto out;
 450
 451         now = ticks;
 452         if (now - sc->ev_stats_update_time < hz)
 453                 goto out;
 454
 455         sc->ev_stats_update_time = now;
 456
 457         /* Add event counts from each event queue in turn */
 458         for (index = 0; index < sc->evq_count; index++) {
 459                 evq = sc->evq[index];
 460                 SFXGE_EVQ_LOCK(evq);
 461                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 462                 SFXGE_EVQ_UNLOCK(evq);
 463         }
 464 out:
 465         SFXGE_ADAPTER_UNLOCK(sc);
 466 }
 467
 468 static int
 469 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 470 {
 471         struct sfxge_softc *sc = arg1;
 472         unsigned int id = arg2;
 473
 474         sfxge_ev_stat_update(sc);
 475
 476         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
 477 }
 478
 479 static void
 480 sfxge_ev_stat_init(struct sfxge_softc *sc)
 481 {
 482         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 483         struct sysctl_oid_list *stat_list;
 484         unsigned int id;
 485         char name[40];
 486
 487         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 488
 489         for (id = 0; id < EV_NQSTATS; id++) {
 490                 snprintf(name, sizeof(name), "ev_%s",
 491                          efx_ev_qstat_name(sc->enp, id));
 492                 SYSCTL_ADD_PROC(
 493                         ctx, stat_list,
 494                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 495                         sc, id, sfxge_ev_stat_handler, "Q",
 496                         "");
 497         }
 498 }
 499
 500 #endif /* EFSYS_OPT_QSTATS */
 501
 502 static void
 503 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 504 {
 505         struct sfxge_evq *evq;
 506         efx_evq_t *eep;
 507
 508         evq = sc->evq[idx];
 509         eep = evq->common;
 510
 511         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 512             ("evq->init_state != SFXGE_EVQ_STARTED"));
 513
 514         (void)efx_ev_qmoderate(eep, us);
 515 }
 516
 517 static int
 518 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 519 {
 520         struct sfxge_softc *sc = arg1;
 521         struct sfxge_intr *intr = &sc->intr;
 522         unsigned int moderation;
 523         int error;
 524         unsigned int index;
 525
 526         SFXGE_ADAPTER_LOCK(sc);
 527
 528         if (req->newptr != NULL) {
 529                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 530                     != 0)
 531                         goto out;
 532
 533                 /* We may not be calling efx_ev_qmoderate() now,
 534                  * so we have to range-check the value ourselves.
 535                  */
 536                 if (moderation >
 537                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
 538                         error = EINVAL;
 539                         goto out;
 540                 }
 541
 542                 sc->ev_moderation = moderation;
 543                 if (intr->state == SFXGE_INTR_STARTED) {
 544                         for (index = 0; index < sc->evq_count; index++)
 545                                 sfxge_ev_qmoderate(sc, index, moderation);
 546                 }
 547         } else {
 548                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 549                                    sizeof(sc->ev_moderation));
 550         }
 551
 552 out:
 553         SFXGE_ADAPTER_UNLOCK(sc);
 554
 555         return (error);
 556 }
 557
 558 static boolean_t
 559 sfxge_ev_initialized(void *arg)
 560 {
 561         struct sfxge_evq *evq;
 562
 563         evq = (struct sfxge_evq *)arg;
 564         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 565
 566         /* Init done events may be duplicated on 7xxx */
 567         KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
 568                 evq->init_state == SFXGE_EVQ_STARTED,
 569             ("evq not starting"));
 570
 571         evq->init_state = SFXGE_EVQ_STARTED;
 572
 573         return (0);
 574 }
 575
 576 static boolean_t
 577 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 578 {
 579         struct sfxge_evq *evq;
 580         struct sfxge_softc *sc;
 581
 582         evq = (struct sfxge_evq *)arg;
 583         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 584
 585         sc = evq->sc;
 586
 587         sfxge_mac_link_update(sc, link_mode);
 588
 589         return (0);
 590 }
 591
 592 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 593         .eec_initialized        = sfxge_ev_initialized,
 594         .eec_rx                 = sfxge_ev_rx,
 595         .eec_tx                 = sfxge_ev_tx,
 596         .eec_exception          = sfxge_ev_exception,
 597         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 598         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 599         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 600         .eec_software           = sfxge_ev_software,
 601         .eec_sram               = sfxge_ev_sram,
 602         .eec_wake_up            = sfxge_ev_wake_up,
 603         .eec_timer              = sfxge_ev_timer,
 604         .eec_link_change        = sfxge_ev_link_change,
 605 };
 606
 607
 608 int
 609 sfxge_ev_qpoll(struct sfxge_evq *evq)
 610 {
 611         int rc;
 612
 613         SFXGE_EVQ_LOCK(evq);
 614
 615         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
 616                             evq->init_state != SFXGE_EVQ_STARTED)) {
 617                 rc = EINVAL;
 618                 goto fail;
 619         }
 620
 621         /* Synchronize the DMA memory for reading */
 622         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 623             BUS_DMASYNC_POSTREAD);
 624
 625         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 626         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 627         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 628         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 629
 630         /* Poll the queue */
 631         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 632
 633         evq->rx_done = 0;
 634         evq->tx_done = 0;
 635
 636         /* Perform any pending completion processing */
 637         sfxge_ev_qcomplete(evq, B_TRUE);
 638
 639         /* Re-prime the event queue for interrupts */
 640         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 641                 goto fail;
 642
 643         SFXGE_EVQ_UNLOCK(evq);
 644
 645         return (0);
 646
 647 fail:
 648         SFXGE_EVQ_UNLOCK(evq);
 649         return (rc);
 650 }
 651
 652 static void
 653 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 654 {
 655         struct sfxge_evq *evq;
 656
 657         evq = sc->evq[index];
 658
 659         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 660             ("evq->init_state != SFXGE_EVQ_STARTED"));
 661
 662         SFXGE_EVQ_LOCK(evq);
 663         evq->init_state = SFXGE_EVQ_INITIALIZED;
 664         evq->read_ptr = 0;
 665         evq->exception = B_FALSE;
 666
 667 #if EFSYS_OPT_QSTATS
 668         /* Add event counts before discarding the common evq state */
 669         efx_ev_qstats_update(evq->common, sc->ev_stats);
 670 #endif
 671
 672         efx_ev_qdestroy(evq->common);
 673         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 674             EFX_EVQ_NBUFS(evq->entries));
 675         SFXGE_EVQ_UNLOCK(evq);
 676 }
 677
 678 static int
 679 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 680 {
 681         struct sfxge_evq *evq;
 682         efsys_mem_t *esmp;
 683         int count;
 684         int rc;
 685
 686         evq = sc->evq[index];
 687         esmp = &evq->mem;
 688
 689         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 690             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 691
 692         /* Clear all events. */
 693         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
 694
 695         /* Program the buffer table. */
 696         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 697             EFX_EVQ_NBUFS(evq->entries))) != 0)
 698                 return (rc);
 699
 700         /* Create the common code event queue. */
 701         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
 702             evq->buf_base_id, sc->ev_moderation, &evq->common)) != 0)
 703                 goto fail;
 704
 705         SFXGE_EVQ_LOCK(evq);
 706
 707         /* Prime the event queue for interrupts */
 708         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 709                 goto fail2;
 710
 711         evq->init_state = SFXGE_EVQ_STARTING;
 712
 713         SFXGE_EVQ_UNLOCK(evq);
 714
 715         /* Wait for the initialization event */
 716         count = 0;
 717         do {
 718                 /* Pause for 100 ms */
 719                 pause("sfxge evq init", hz / 10);
 720
 721                 /* Check to see if the test event has been processed */
 722                 if (evq->init_state == SFXGE_EVQ_STARTED)
 723                         goto done;
 724
 725         } while (++count < 20);
 726
 727         rc = ETIMEDOUT;
 728         goto fail3;
 729
 730 done:
 731         return (0);
 732
 733 fail3:
 734         SFXGE_EVQ_LOCK(evq);
 735         evq->init_state = SFXGE_EVQ_INITIALIZED;
 736 fail2:
 737         SFXGE_EVQ_UNLOCK(evq);
 738         efx_ev_qdestroy(evq->common);
 739 fail:
 740         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 741             EFX_EVQ_NBUFS(evq->entries));
 742
 743         return (rc);
 744 }
 745
 746 void
 747 sfxge_ev_stop(struct sfxge_softc *sc)
 748 {
 749         struct sfxge_intr *intr;
 750         efx_nic_t *enp;
 751         int index;
 752
 753         intr = &sc->intr;
 754         enp = sc->enp;
 755
 756         KASSERT(intr->state == SFXGE_INTR_STARTED,
 757             ("Interrupts not started"));
 758
 759         /* Stop the event queue(s) */
 760         index = sc->evq_count;
 761         while (--index >= 0)
 762                 sfxge_ev_qstop(sc, index);
 763
 764         /* Tear down the event module */
 765         efx_ev_fini(enp);
 766 }
 767
 768 int
 769 sfxge_ev_start(struct sfxge_softc *sc)
 770 {
 771         struct sfxge_intr *intr;
 772         int index;
 773         int rc;
 774
 775         intr = &sc->intr;
 776
 777         KASSERT(intr->state == SFXGE_INTR_STARTED,
 778             ("intr->state != SFXGE_INTR_STARTED"));
 779
 780         /* Initialize the event module */
 781         if ((rc = efx_ev_init(sc->enp)) != 0)
 782                 return (rc);
 783
 784         /* Start the event queues */
 785         for (index = 0; index < sc->evq_count; index++) {
 786                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 787                         goto fail;
 788         }
 789
 790         return (0);
 791
 792 fail:
 793         /* Stop the event queue(s) */
 794         while (--index >= 0)
 795                 sfxge_ev_qstop(sc, index);
 796
 797         /* Tear down the event module */
 798         efx_ev_fini(sc->enp);
 799
 800         return (rc);
 801 }
 802
 803 static void
 804 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 805 {
 806         struct sfxge_evq *evq;
 807
 808         evq = sc->evq[index];
 809
 810         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 811             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 812         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 813
 814         sfxge_dma_free(&evq->mem);
 815
 816         sc->evq[index] = NULL;
 817
 818         SFXGE_EVQ_LOCK_DESTROY(evq);
 819
 820         free(evq, M_SFXGE);
 821 }
 822
 823 static int
 824 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 825 {
 826         struct sfxge_evq *evq;
 827         efsys_mem_t *esmp;
 828         int rc;
 829
 830         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 831
 832         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 833         evq->sc = sc;
 834         evq->index = index;
 835         sc->evq[index] = evq;
 836         esmp = &evq->mem;
 837
 838         /* Build an event queue with room for one event per tx and rx buffer,
 839          * plus some extra for link state events and MCDI completions.
 840          * There are three tx queues in the first event queue and one in
 841          * other.
 842          */
 843         if (index == 0)
 844                 evq->entries =
 845                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 846                                            3 * sc->txq_entries +
 847                                            128);
 848         else
 849                 evq->entries =
 850                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 851                                            sc->txq_entries +
 852                                            128);
 853
 854         /* Initialise TX completion list */
 855         evq->txqs = &evq->txq;
 856
 857         /* Allocate DMA space. */
 858         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
 859                 return (rc);
 860
 861         /* Allocate buffer table entries. */
 862         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
 863                                  &evq->buf_base_id);
 864
 865         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
 866
 867         evq->init_state = SFXGE_EVQ_INITIALIZED;
 868
 869         return (0);
 870 }
 871
 872 void
 873 sfxge_ev_fini(struct sfxge_softc *sc)
 874 {
 875         struct sfxge_intr *intr;
 876         int index;
 877
 878         intr = &sc->intr;
 879
 880         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 881             ("intr->state != SFXGE_INTR_INITIALIZED"));
 882
 883         sc->ev_moderation = 0;
 884
 885         /* Tear down the event queue(s). */
 886         index = sc->evq_count;
 887         while (--index >= 0)
 888                 sfxge_ev_qfini(sc, index);
 889
 890         sc->evq_count = 0;
 891 }
 892
 893 int
 894 sfxge_ev_init(struct sfxge_softc *sc)
 895 {
 896         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 897         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 898         struct sfxge_intr *intr;
 899         int index;
 900         int rc;
 901
 902         intr = &sc->intr;
 903
 904         sc->evq_count = intr->n_alloc;
 905
 906         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 907             ("intr->state != SFXGE_INTR_INITIALIZED"));
 908
 909         /* Set default interrupt moderation; add a sysctl to
 910          * read and change it.
 911          */
 912         sc->ev_moderation = SFXGE_MODERATION;
 913         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 914                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 915                         sc, 0, sfxge_int_mod_handler, "IU",
 916                         "sfxge interrupt moderation (us)");
 917
 918         /*
 919          * Initialize the event queue(s) - one per interrupt.
 920          */
 921         for (index = 0; index < sc->evq_count; index++) {
 922                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 923                         goto fail;
 924         }
 925
 926 #if EFSYS_OPT_QSTATS
 927         sfxge_ev_stat_init(sc);
 928 #endif
 929
 930         return (0);
 931
 932 fail:
 933         while (--index >= 0)
 934                 sfxge_ev_qfini(sc, index);
 935
 936         sc->evq_count = 0;
 937         return (rc);
 938 }