sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright notice,
  12  *    this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  *    this list of conditions and the following disclaimer in the documentation
  15  *    and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * The views and conclusions contained in the software and documentation are
  30  * those of the authors and should not be interpreted as representing official
  31  * policies, either expressed or implied, of the FreeBSD Project.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/malloc.h>
  40 #include <sys/param.h>
  41 #include <sys/queue.h>
  42 #include <sys/systm.h>
  43 #include <sys/taskqueue.h>
  44
  45 #include "common/efx.h"
  46
  47 #include "sfxge.h"
  48
  49 static void
  50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  51 {
  52         struct sfxge_softc *sc;
  53         unsigned int index;
  54         struct sfxge_rxq *rxq;
  55         struct sfxge_txq *txq;
  56
  57         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  58
  59         sc = evq->sc;
  60         index = evq->index;
  61         rxq = sc->rxq[index];
  62
  63         if ((txq = evq->txq) != NULL) {
  64                 evq->txq = NULL;
  65                 evq->txqs = &(evq->txq);
  66
  67                 do {
  68                         struct sfxge_txq *next;
  69
  70                         next = txq->next;
  71                         txq->next = NULL;
  72
  73                         KASSERT(txq->evq_index == index,
  74                             ("txq->evq_index != index"));
  75
  76                         if (txq->pending != txq->completed)
  77                                 sfxge_tx_qcomplete(txq, evq);
  78
  79                         txq = next;
  80                 } while (txq != NULL);
  81         }
  82
  83         if (rxq->pending != rxq->completed)
  84                 sfxge_rx_qcomplete(rxq, eop);
  85 }
  86
  87 static struct sfxge_rxq *
  88 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
  89 {
  90         struct sfxge_rxq *rxq;
  91
  92         KASSERT(label == 0, ("unexpected rxq label != 0"));
  93
  94         rxq = evq->sc->rxq[evq->index];
  95
  96         KASSERT(rxq != NULL, ("rxq == NULL"));
  97         KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
  98
  99         return (rxq);
 100 }
 101
 102 static boolean_t
 103 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
 104             uint16_t flags)
 105 {
 106         struct sfxge_evq *evq;
 107         struct sfxge_softc *sc;
 108         struct sfxge_rxq *rxq;
 109         unsigned int stop;
 110         unsigned int delta;
 111         struct sfxge_rx_sw_desc *rx_desc;
 112
 113         evq = arg;
 114         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 115
 116         sc = evq->sc;
 117
 118         if (evq->exception)
 119                 goto done;
 120
 121         rxq = sfxge_get_rxq_by_label(evq, label);
 122         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
 123                 goto done;
 124
 125         stop = (id + 1) & rxq->ptr_mask;
 126         id = rxq->pending & rxq->ptr_mask;
 127         delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
 128         rxq->pending += delta;
 129
 130         if (delta != 1) {
 131                 if ((delta <= 0) ||
 132                     (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
 133                         evq->exception = B_TRUE;
 134
 135                         device_printf(sc->dev, "RX completion out of order"
 136                                                   " (id=%#x delta=%u flags=%#x); resetting\n",
 137                                                   id, delta, flags);
 138                         sfxge_schedule_reset(sc);
 139
 140                         goto done;
 141                 }
 142         }
 143
 144         rx_desc = &rxq->queue[id];
 145
 146         prefetch_read_many(rx_desc->mbuf);
 147
 148         for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
 149                 rx_desc = &rxq->queue[id];
 150                 KASSERT(rx_desc->flags == EFX_DISCARD,
 151                                 ("rx_desc->flags != EFX_DISCARD"));
 152                 rx_desc->flags = flags;
 153
 154                 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 155                 rx_desc->size = (uint16_t)size;
 156         }
 157
 158         evq->rx_done++;
 159
 160         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 161                 sfxge_ev_qcomplete(evq, B_FALSE);
 162
 163 done:
 164         return (evq->rx_done >= SFXGE_EV_BATCH);
 165 }
 166
 167 static boolean_t
 168 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 169 {
 170         struct sfxge_evq *evq;
 171         struct sfxge_softc *sc;
 172
 173         evq = (struct sfxge_evq *)arg;
 174         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 175
 176         sc = evq->sc;
 177
 178         DBGPRINT(sc->dev, "[%d] %s", evq->index,
 179                           (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
 180                           (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
 181                           (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
 182                           (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
 183                           (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
 184                           (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
 185                           (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
 186                           (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
 187                           (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
 188                           "UNKNOWN");
 189
 190         evq->exception = B_TRUE;
 191
 192         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 193                 device_printf(sc->dev,
 194                               "hardware exception (code=%u); resetting\n",
 195                               code);
 196                 sfxge_schedule_reset(sc);
 197         }
 198
 199         return (B_FALSE);
 200 }
 201
 202 static boolean_t
 203 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
 204 {
 205         struct sfxge_evq *evq;
 206         struct sfxge_softc *sc;
 207         struct sfxge_rxq *rxq;
 208         unsigned int index;
 209         uint16_t magic;
 210
 211         evq = (struct sfxge_evq *)arg;
 212         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 213
 214         sc = evq->sc;
 215         rxq = sc->rxq[rxq_index];
 216
 217         KASSERT(rxq != NULL, ("rxq == NULL"));
 218
 219         /* Resend a software event on the correct queue */
 220         index = rxq->index;
 221         if (index == evq->index) {
 222                 sfxge_rx_qflush_done(rxq);
 223                 return (B_FALSE);
 224         }
 225
 226         evq = sc->evq[index];
 227         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
 228
 229         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 230             ("evq not started"));
 231         efx_ev_qpost(evq->common, magic);
 232
 233         return (B_FALSE);
 234 }
 235
 236 static boolean_t
 237 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
 238 {
 239         struct sfxge_evq *evq;
 240         struct sfxge_softc *sc;
 241         struct sfxge_rxq *rxq;
 242         unsigned int index;
 243         uint16_t magic;
 244
 245         evq = (struct sfxge_evq *)arg;
 246         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 247
 248         sc = evq->sc;
 249         rxq = sc->rxq[rxq_index];
 250
 251         KASSERT(rxq != NULL, ("rxq == NULL"));
 252
 253         /* Resend a software event on the correct queue */
 254         index = rxq->index;
 255         evq = sc->evq[index];
 256         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
 257
 258         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 259             ("evq not started"));
 260         efx_ev_qpost(evq->common, magic);
 261
 262         return (B_FALSE);
 263 }
 264
 265 static struct sfxge_txq *
 266 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
 267 {
 268         unsigned int index;
 269
 270         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
 271             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
 272         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
 273         return (evq->sc->txq[index]);
 274 }
 275
 276 static boolean_t
 277 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 278 {
 279         struct sfxge_evq *evq;
 280         struct sfxge_txq *txq;
 281         unsigned int stop;
 282         unsigned int delta;
 283
 284         evq = (struct sfxge_evq *)arg;
 285         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 286
 287         txq = sfxge_get_txq_by_label(evq, label);
 288
 289         KASSERT(txq != NULL, ("txq == NULL"));
 290         KASSERT(evq->index == txq->evq_index,
 291             ("evq->index != txq->evq_index"));
 292
 293         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
 294                 goto done;
 295
 296         stop = (id + 1) & txq->ptr_mask;
 297         id = txq->pending & txq->ptr_mask;
 298
 299         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
 300         txq->pending += delta;
 301
 302         evq->tx_done++;
 303
 304         if (txq->next == NULL &&
 305             evq->txqs != &(txq->next)) {
 306                 *(evq->txqs) = txq;
 307                 evq->txqs = &(txq->next);
 308         }
 309
 310         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 311                 sfxge_tx_qcomplete(txq, evq);
 312
 313 done:
 314         return (evq->tx_done >= SFXGE_EV_BATCH);
 315 }
 316
 317 static boolean_t
 318 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
 319 {
 320         struct sfxge_evq *evq;
 321         struct sfxge_softc *sc;
 322         struct sfxge_txq *txq;
 323         uint16_t magic;
 324
 325         evq = (struct sfxge_evq *)arg;
 326         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 327
 328         sc = evq->sc;
 329         txq = sc->txq[txq_index];
 330
 331         KASSERT(txq != NULL, ("txq == NULL"));
 332         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 333             ("txq not initialized"));
 334
 335         if (txq->evq_index == evq->index) {
 336                 sfxge_tx_qflush_done(txq);
 337                 return (B_FALSE);
 338         }
 339
 340         /* Resend a software event on the correct queue */
 341         evq = sc->evq[txq->evq_index];
 342         magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
 343
 344         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 345             ("evq not started"));
 346         efx_ev_qpost(evq->common, magic);
 347
 348         return (B_FALSE);
 349 }
 350
 351 static boolean_t
 352 sfxge_ev_software(void *arg, uint16_t magic)
 353 {
 354         struct sfxge_evq *evq;
 355         struct sfxge_softc *sc;
 356         unsigned int label;
 357
 358         evq = (struct sfxge_evq *)arg;
 359         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 360
 361         sc = evq->sc;
 362
 363         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 364         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 365
 366         switch (magic) {
 367         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
 368                 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
 369                 break;
 370
 371         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
 372                 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
 373                 break;
 374
 375         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
 376                 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
 377                 break;
 378
 379         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
 380                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
 381
 382                 KASSERT(txq != NULL, ("txq == NULL"));
 383                 KASSERT(evq->index == txq->evq_index,
 384                     ("evq->index != txq->evq_index"));
 385
 386                 sfxge_tx_qflush_done(txq);
 387                 break;
 388         }
 389         default:
 390                 break;
 391         }
 392
 393         return (B_FALSE);
 394 }
 395
 396 static boolean_t
 397 sfxge_ev_sram(void *arg, uint32_t code)
 398 {
 399         (void)arg;
 400         (void)code;
 401
 402         switch (code) {
 403         case EFX_SRAM_UPDATE:
 404                 EFSYS_PROBE(sram_update);
 405                 break;
 406
 407         case EFX_SRAM_CLEAR:
 408                 EFSYS_PROBE(sram_clear);
 409                 break;
 410
 411         case EFX_SRAM_ILLEGAL_CLEAR:
 412                 EFSYS_PROBE(sram_illegal_clear);
 413                 break;
 414
 415         default:
 416                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 417                 break;
 418         }
 419
 420         return (B_FALSE);
 421 }
 422
 423 static boolean_t
 424 sfxge_ev_timer(void *arg, uint32_t index)
 425 {
 426         (void)arg;
 427         (void)index;
 428
 429         return (B_FALSE);
 430 }
 431
 432 static boolean_t
 433 sfxge_ev_wake_up(void *arg, uint32_t index)
 434 {
 435         (void)arg;
 436         (void)index;
 437
 438         return (B_FALSE);
 439 }
 440
 441 #if EFSYS_OPT_QSTATS
 442
 443 static void
 444 sfxge_ev_stat_update(struct sfxge_softc *sc)
 445 {
 446         struct sfxge_evq *evq;
 447         unsigned int index;
 448         clock_t now;
 449
 450         SFXGE_ADAPTER_LOCK(sc);
 451
 452         if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
 453                 goto out;
 454
 455         now = ticks;
 456         if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz)
 457                 goto out;
 458
 459         sc->ev_stats_update_time = now;
 460
 461         /* Add event counts from each event queue in turn */
 462         for (index = 0; index < sc->evq_count; index++) {
 463                 evq = sc->evq[index];
 464                 SFXGE_EVQ_LOCK(evq);
 465                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 466                 SFXGE_EVQ_UNLOCK(evq);
 467         }
 468 out:
 469         SFXGE_ADAPTER_UNLOCK(sc);
 470 }
 471
 472 static int
 473 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 474 {
 475         struct sfxge_softc *sc = arg1;
 476         unsigned int id = arg2;
 477
 478         sfxge_ev_stat_update(sc);
 479
 480         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
 481 }
 482
 483 static void
 484 sfxge_ev_stat_init(struct sfxge_softc *sc)
 485 {
 486         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 487         struct sysctl_oid_list *stat_list;
 488         unsigned int id;
 489         char name[40];
 490
 491         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 492
 493         for (id = 0; id < EV_NQSTATS; id++) {
 494                 snprintf(name, sizeof(name), "ev_%s",
 495                          efx_ev_qstat_name(sc->enp, id));
 496                 SYSCTL_ADD_PROC(
 497                         ctx, stat_list,
 498                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 499                         sc, id, sfxge_ev_stat_handler, "Q",
 500                         "");
 501         }
 502 }
 503
 504 #endif /* EFSYS_OPT_QSTATS */
 505
 506 static void
 507 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 508 {
 509         struct sfxge_evq *evq;
 510         efx_evq_t *eep;
 511
 512         evq = sc->evq[idx];
 513         eep = evq->common;
 514
 515         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 516             ("evq->init_state != SFXGE_EVQ_STARTED"));
 517
 518         (void)efx_ev_qmoderate(eep, us);
 519 }
 520
 521 static int
 522 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 523 {
 524         struct sfxge_softc *sc = arg1;
 525         struct sfxge_intr *intr = &sc->intr;
 526         unsigned int moderation;
 527         int error;
 528         unsigned int index;
 529
 530         SFXGE_ADAPTER_LOCK(sc);
 531
 532         if (req->newptr != NULL) {
 533                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 534                     != 0)
 535                         goto out;
 536
 537                 /* We may not be calling efx_ev_qmoderate() now,
 538                  * so we have to range-check the value ourselves.
 539                  */
 540                 if (moderation >
 541                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
 542                         error = EINVAL;
 543                         goto out;
 544                 }
 545
 546                 sc->ev_moderation = moderation;
 547                 if (intr->state == SFXGE_INTR_STARTED) {
 548                         for (index = 0; index < sc->evq_count; index++)
 549                                 sfxge_ev_qmoderate(sc, index, moderation);
 550                 }
 551         } else {
 552                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 553                                    sizeof(sc->ev_moderation));
 554         }
 555
 556 out:
 557         SFXGE_ADAPTER_UNLOCK(sc);
 558
 559         return (error);
 560 }
 561
 562 static boolean_t
 563 sfxge_ev_initialized(void *arg)
 564 {
 565         struct sfxge_evq *evq;
 566
 567         evq = (struct sfxge_evq *)arg;
 568         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 569
 570         /* Init done events may be duplicated on 7xxx */
 571         KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
 572                 evq->init_state == SFXGE_EVQ_STARTED,
 573             ("evq not starting"));
 574
 575         evq->init_state = SFXGE_EVQ_STARTED;
 576
 577         return (0);
 578 }
 579
 580 static boolean_t
 581 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 582 {
 583         struct sfxge_evq *evq;
 584         struct sfxge_softc *sc;
 585
 586         evq = (struct sfxge_evq *)arg;
 587         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 588
 589         sc = evq->sc;
 590
 591         sfxge_mac_link_update(sc, link_mode);
 592
 593         return (0);
 594 }
 595
 596 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 597         .eec_initialized        = sfxge_ev_initialized,
 598         .eec_rx                 = sfxge_ev_rx,
 599         .eec_tx                 = sfxge_ev_tx,
 600         .eec_exception          = sfxge_ev_exception,
 601         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 602         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 603         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 604         .eec_software           = sfxge_ev_software,
 605         .eec_sram               = sfxge_ev_sram,
 606         .eec_wake_up            = sfxge_ev_wake_up,
 607         .eec_timer              = sfxge_ev_timer,
 608         .eec_link_change        = sfxge_ev_link_change,
 609 };
 610
 611
 612 int
 613 sfxge_ev_qpoll(struct sfxge_evq *evq)
 614 {
 615         int rc;
 616
 617         SFXGE_EVQ_LOCK(evq);
 618
 619         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
 620                             evq->init_state != SFXGE_EVQ_STARTED)) {
 621                 rc = EINVAL;
 622                 goto fail;
 623         }
 624
 625         /* Synchronize the DMA memory for reading */
 626         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 627             BUS_DMASYNC_POSTREAD);
 628
 629         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 630         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 631         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 632         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 633
 634         /* Poll the queue */
 635         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 636
 637         evq->rx_done = 0;
 638         evq->tx_done = 0;
 639
 640         /* Perform any pending completion processing */
 641         sfxge_ev_qcomplete(evq, B_TRUE);
 642
 643         /* Re-prime the event queue for interrupts */
 644         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 645                 goto fail;
 646
 647         SFXGE_EVQ_UNLOCK(evq);
 648
 649         return (0);
 650
 651 fail:
 652         SFXGE_EVQ_UNLOCK(evq);
 653         return (rc);
 654 }
 655
 656 static void
 657 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 658 {
 659         struct sfxge_evq *evq;
 660
 661         evq = sc->evq[index];
 662
 663         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 664             ("evq->init_state != SFXGE_EVQ_STARTED"));
 665
 666         SFXGE_EVQ_LOCK(evq);
 667         evq->init_state = SFXGE_EVQ_INITIALIZED;
 668         evq->read_ptr = 0;
 669         evq->exception = B_FALSE;
 670
 671 #if EFSYS_OPT_QSTATS
 672         /* Add event counts before discarding the common evq state */
 673         efx_ev_qstats_update(evq->common, sc->ev_stats);
 674 #endif
 675
 676         efx_ev_qdestroy(evq->common);
 677         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 678             EFX_EVQ_NBUFS(evq->entries));
 679         SFXGE_EVQ_UNLOCK(evq);
 680 }
 681
 682 static int
 683 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 684 {
 685         struct sfxge_evq *evq;
 686         efsys_mem_t *esmp;
 687         int count;
 688         int rc;
 689
 690         evq = sc->evq[index];
 691         esmp = &evq->mem;
 692
 693         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 694             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 695
 696         /* Clear all events. */
 697         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
 698
 699         /* Program the buffer table. */
 700         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 701             EFX_EVQ_NBUFS(evq->entries))) != 0)
 702                 return (rc);
 703
 704         /* Create the common code event queue. */
 705         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
 706             evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
 707             &evq->common)) != 0)
 708                 goto fail;
 709
 710         SFXGE_EVQ_LOCK(evq);
 711
 712         /* Prime the event queue for interrupts */
 713         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 714                 goto fail2;
 715
 716         evq->init_state = SFXGE_EVQ_STARTING;
 717
 718         SFXGE_EVQ_UNLOCK(evq);
 719
 720         /* Wait for the initialization event */
 721         count = 0;
 722         do {
 723                 /* Pause for 100 ms */
 724                 pause("sfxge evq init", hz / 10);
 725
 726                 /* Check to see if the test event has been processed */
 727                 if (evq->init_state == SFXGE_EVQ_STARTED)
 728                         goto done;
 729
 730         } while (++count < 20);
 731
 732         rc = ETIMEDOUT;
 733         goto fail3;
 734
 735 done:
 736         return (0);
 737
 738 fail3:
 739         SFXGE_EVQ_LOCK(evq);
 740         evq->init_state = SFXGE_EVQ_INITIALIZED;
 741 fail2:
 742         SFXGE_EVQ_UNLOCK(evq);
 743         efx_ev_qdestroy(evq->common);
 744 fail:
 745         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 746             EFX_EVQ_NBUFS(evq->entries));
 747
 748         return (rc);
 749 }
 750
 751 void
 752 sfxge_ev_stop(struct sfxge_softc *sc)
 753 {
 754         struct sfxge_intr *intr;
 755         efx_nic_t *enp;
 756         int index;
 757
 758         intr = &sc->intr;
 759         enp = sc->enp;
 760
 761         KASSERT(intr->state == SFXGE_INTR_STARTED,
 762             ("Interrupts not started"));
 763
 764         /* Stop the event queue(s) */
 765         index = sc->evq_count;
 766         while (--index >= 0)
 767                 sfxge_ev_qstop(sc, index);
 768
 769         /* Tear down the event module */
 770         efx_ev_fini(enp);
 771 }
 772
 773 int
 774 sfxge_ev_start(struct sfxge_softc *sc)
 775 {
 776         struct sfxge_intr *intr;
 777         int index;
 778         int rc;
 779
 780         intr = &sc->intr;
 781
 782         KASSERT(intr->state == SFXGE_INTR_STARTED,
 783             ("intr->state != SFXGE_INTR_STARTED"));
 784
 785         /* Initialize the event module */
 786         if ((rc = efx_ev_init(sc->enp)) != 0)
 787                 return (rc);
 788
 789         /* Start the event queues */
 790         for (index = 0; index < sc->evq_count; index++) {
 791                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 792                         goto fail;
 793         }
 794
 795         return (0);
 796
 797 fail:
 798         /* Stop the event queue(s) */
 799         while (--index >= 0)
 800                 sfxge_ev_qstop(sc, index);
 801
 802         /* Tear down the event module */
 803         efx_ev_fini(sc->enp);
 804
 805         return (rc);
 806 }
 807
 808 static void
 809 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 810 {
 811         struct sfxge_evq *evq;
 812
 813         evq = sc->evq[index];
 814
 815         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 816             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 817         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 818
 819         sfxge_dma_free(&evq->mem);
 820
 821         sc->evq[index] = NULL;
 822
 823         SFXGE_EVQ_LOCK_DESTROY(evq);
 824
 825         free(evq, M_SFXGE);
 826 }
 827
 828 static int
 829 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 830 {
 831         struct sfxge_evq *evq;
 832         efsys_mem_t *esmp;
 833         int rc;
 834
 835         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 836
 837         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 838         evq->sc = sc;
 839         evq->index = index;
 840         sc->evq[index] = evq;
 841         esmp = &evq->mem;
 842
 843         /* Build an event queue with room for one event per tx and rx buffer,
 844          * plus some extra for link state events and MCDI completions.
 845          * There are three tx queues in the first event queue and one in
 846          * other.
 847          */
 848         if (index == 0)
 849                 evq->entries =
 850                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 851                                            3 * sc->txq_entries +
 852                                            128);
 853         else
 854                 evq->entries =
 855                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 856                                            sc->txq_entries +
 857                                            128);
 858
 859         /* Initialise TX completion list */
 860         evq->txqs = &evq->txq;
 861
 862         /* Allocate DMA space. */
 863         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
 864                 return (rc);
 865
 866         /* Allocate buffer table entries. */
 867         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
 868                                  &evq->buf_base_id);
 869
 870         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
 871
 872         evq->init_state = SFXGE_EVQ_INITIALIZED;
 873
 874         return (0);
 875 }
 876
 877 void
 878 sfxge_ev_fini(struct sfxge_softc *sc)
 879 {
 880         struct sfxge_intr *intr;
 881         int index;
 882
 883         intr = &sc->intr;
 884
 885         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 886             ("intr->state != SFXGE_INTR_INITIALIZED"));
 887
 888         sc->ev_moderation = 0;
 889
 890         /* Tear down the event queue(s). */
 891         index = sc->evq_count;
 892         while (--index >= 0)
 893                 sfxge_ev_qfini(sc, index);
 894
 895         sc->evq_count = 0;
 896 }
 897
 898 int
 899 sfxge_ev_init(struct sfxge_softc *sc)
 900 {
 901         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 902         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 903         struct sfxge_intr *intr;
 904         int index;
 905         int rc;
 906
 907         intr = &sc->intr;
 908
 909         sc->evq_count = intr->n_alloc;
 910
 911         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 912             ("intr->state != SFXGE_INTR_INITIALIZED"));
 913
 914         /* Set default interrupt moderation; add a sysctl to
 915          * read and change it.
 916          */
 917         sc->ev_moderation = SFXGE_MODERATION;
 918         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 919                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 920                         sc, 0, sfxge_int_mod_handler, "IU",
 921                         "sfxge interrupt moderation (us)");
 922
 923         /*
 924          * Initialize the event queue(s) - one per interrupt.
 925          */
 926         for (index = 0; index < sc->evq_count; index++) {
 927                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 928                         goto fail;
 929         }
 930
 931 #if EFSYS_OPT_QSTATS
 932         sfxge_ev_stat_init(sc);
 933 #endif
 934
 935         return (0);
 936
 937 fail:
 938         while (--index >= 0)
 939                 sfxge_ev_qfini(sc, index);
 940
 941         sc->evq_count = 0;
 942         return (rc);
 943 }