sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright notice,
  12  *    this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  *    this list of conditions and the following disclaimer in the documentation
  15  *    and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * The views and conclusions contained in the software and documentation are
  30  * those of the authors and should not be interpreted as representing official
  31  * policies, either expressed or implied, of the FreeBSD Project.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39
  40 #include "common/efx.h"
  41
  42 #include "sfxge.h"
  43
  44 static void
  45 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  46 {
  47         struct sfxge_softc *sc;
  48         unsigned int index;
  49         struct sfxge_rxq *rxq;
  50         struct sfxge_txq *txq;
  51
  52         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  53
  54         sc = evq->sc;
  55         index = evq->index;
  56         rxq = sc->rxq[index];
  57
  58         if ((txq = evq->txq) != NULL) {
  59                 evq->txq = NULL;
  60                 evq->txqs = &(evq->txq);
  61
  62                 do {
  63                         struct sfxge_txq *next;
  64
  65                         next = txq->next;
  66                         txq->next = NULL;
  67
  68                         KASSERT(txq->evq_index == index,
  69                             ("txq->evq_index != index"));
  70
  71                         if (txq->pending != txq->completed)
  72                                 sfxge_tx_qcomplete(txq, evq);
  73
  74                         txq = next;
  75                 } while (txq != NULL);
  76         }
  77
  78         if (rxq->pending != rxq->completed)
  79                 sfxge_rx_qcomplete(rxq, eop);
  80 }
  81
  82 static struct sfxge_rxq *
  83 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
  84 {
  85         struct sfxge_rxq *rxq;
  86
  87         KASSERT(label == 0, ("unexpected rxq label != 0"));
  88
  89         rxq = evq->sc->rxq[evq->index];
  90
  91         KASSERT(rxq != NULL, ("rxq == NULL"));
  92         KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
  93
  94         return (rxq);
  95 }
  96
  97 static boolean_t
  98 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  99             uint16_t flags)
 100 {
 101         struct sfxge_evq *evq;
 102         struct sfxge_softc *sc;
 103         struct sfxge_rxq *rxq;
 104         unsigned int stop;
 105         unsigned int delta;
 106         struct sfxge_rx_sw_desc *rx_desc;
 107
 108         evq = arg;
 109         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 110
 111         sc = evq->sc;
 112
 113         if (evq->exception)
 114                 goto done;
 115
 116         rxq = sfxge_get_rxq_by_label(evq, label);
 117         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
 118                 goto done;
 119
 120         stop = (id + 1) & rxq->ptr_mask;
 121         id = rxq->pending & rxq->ptr_mask;
 122         delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
 123         rxq->pending += delta;
 124
 125         if (delta != 1) {
 126                 if ((delta <= 0) ||
 127                     (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
 128                         evq->exception = B_TRUE;
 129
 130                         device_printf(sc->dev, "RX completion out of order"
 131                                                   " (id=%#x delta=%u flags=%#x); resetting\n",
 132                                                   id, delta, flags);
 133                         sfxge_schedule_reset(sc);
 134
 135                         goto done;
 136                 }
 137         }
 138
 139         rx_desc = &rxq->queue[id];
 140
 141         prefetch_read_many(rx_desc->mbuf);
 142
 143         for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
 144                 rx_desc = &rxq->queue[id];
 145                 KASSERT(rx_desc->flags == EFX_DISCARD,
 146                                 ("rx_desc->flags != EFX_DISCARD"));
 147                 rx_desc->flags = flags;
 148
 149                 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 150                 rx_desc->size = (uint16_t)size;
 151         }
 152
 153         evq->rx_done++;
 154
 155         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 156                 sfxge_ev_qcomplete(evq, B_FALSE);
 157
 158 done:
 159         return (evq->rx_done >= SFXGE_EV_BATCH);
 160 }
 161
 162 static boolean_t
 163 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 164 {
 165         struct sfxge_evq *evq;
 166         struct sfxge_softc *sc;
 167
 168         evq = (struct sfxge_evq *)arg;
 169         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 170
 171         sc = evq->sc;
 172
 173         DBGPRINT(sc->dev, "[%d] %s", evq->index,
 174                           (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
 175                           (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
 176                           (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
 177                           (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
 178                           (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
 179                           (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
 180                           (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
 181                           (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
 182                           (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
 183                           "UNKNOWN");
 184
 185         evq->exception = B_TRUE;
 186
 187         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 188                 device_printf(sc->dev,
 189                               "hardware exception (code=%u); resetting\n",
 190                               code);
 191                 sfxge_schedule_reset(sc);
 192         }
 193
 194         return (B_FALSE);
 195 }
 196
 197 static boolean_t
 198 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
 199 {
 200         struct sfxge_evq *evq;
 201         struct sfxge_softc *sc;
 202         struct sfxge_rxq *rxq;
 203         unsigned int index;
 204         uint16_t magic;
 205
 206         evq = (struct sfxge_evq *)arg;
 207         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 208
 209         sc = evq->sc;
 210         rxq = sc->rxq[rxq_index];
 211
 212         KASSERT(rxq != NULL, ("rxq == NULL"));
 213
 214         /* Resend a software event on the correct queue */
 215         index = rxq->index;
 216         if (index == evq->index) {
 217                 sfxge_rx_qflush_done(rxq);
 218                 return (B_FALSE);
 219         }
 220
 221         evq = sc->evq[index];
 222         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
 223
 224         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 225             ("evq not started"));
 226         efx_ev_qpost(evq->common, magic);
 227
 228         return (B_FALSE);
 229 }
 230
 231 static boolean_t
 232 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
 233 {
 234         struct sfxge_evq *evq;
 235         struct sfxge_softc *sc;
 236         struct sfxge_rxq *rxq;
 237         unsigned int index;
 238         uint16_t magic;
 239
 240         evq = (struct sfxge_evq *)arg;
 241         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 242
 243         sc = evq->sc;
 244         rxq = sc->rxq[rxq_index];
 245
 246         KASSERT(rxq != NULL, ("rxq == NULL"));
 247
 248         /* Resend a software event on the correct queue */
 249         index = rxq->index;
 250         evq = sc->evq[index];
 251         magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
 252
 253         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 254             ("evq not started"));
 255         efx_ev_qpost(evq->common, magic);
 256
 257         return (B_FALSE);
 258 }
 259
 260 static struct sfxge_txq *
 261 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
 262 {
 263         unsigned int index;
 264
 265         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
 266             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
 267         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
 268         return (evq->sc->txq[index]);
 269 }
 270
 271 static boolean_t
 272 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 273 {
 274         struct sfxge_evq *evq;
 275         struct sfxge_txq *txq;
 276         unsigned int stop;
 277         unsigned int delta;
 278
 279         evq = (struct sfxge_evq *)arg;
 280         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 281
 282         txq = sfxge_get_txq_by_label(evq, label);
 283
 284         KASSERT(txq != NULL, ("txq == NULL"));
 285         KASSERT(evq->index == txq->evq_index,
 286             ("evq->index != txq->evq_index"));
 287
 288         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
 289                 goto done;
 290
 291         stop = (id + 1) & txq->ptr_mask;
 292         id = txq->pending & txq->ptr_mask;
 293
 294         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
 295         txq->pending += delta;
 296
 297         evq->tx_done++;
 298
 299         if (txq->next == NULL &&
 300             evq->txqs != &(txq->next)) {
 301                 *(evq->txqs) = txq;
 302                 evq->txqs = &(txq->next);
 303         }
 304
 305         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 306                 sfxge_tx_qcomplete(txq, evq);
 307
 308 done:
 309         return (evq->tx_done >= SFXGE_EV_BATCH);
 310 }
 311
 312 static boolean_t
 313 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
 314 {
 315         struct sfxge_evq *evq;
 316         struct sfxge_softc *sc;
 317         struct sfxge_txq *txq;
 318         uint16_t magic;
 319
 320         evq = (struct sfxge_evq *)arg;
 321         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 322
 323         sc = evq->sc;
 324         txq = sc->txq[txq_index];
 325
 326         KASSERT(txq != NULL, ("txq == NULL"));
 327         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 328             ("txq not initialized"));
 329
 330         if (txq->evq_index == evq->index) {
 331                 sfxge_tx_qflush_done(txq);
 332                 return (B_FALSE);
 333         }
 334
 335         /* Resend a software event on the correct queue */
 336         evq = sc->evq[txq->evq_index];
 337         magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
 338
 339         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 340             ("evq not started"));
 341         efx_ev_qpost(evq->common, magic);
 342
 343         return (B_FALSE);
 344 }
 345
 346 static boolean_t
 347 sfxge_ev_software(void *arg, uint16_t magic)
 348 {
 349         struct sfxge_evq *evq;
 350         struct sfxge_softc *sc;
 351         unsigned int label;
 352
 353         evq = (struct sfxge_evq *)arg;
 354         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 355
 356         sc = evq->sc;
 357
 358         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 359         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 360
 361         switch (magic) {
 362         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
 363                 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
 364                 break;
 365
 366         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
 367                 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
 368                 break;
 369
 370         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
 371                 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
 372                 break;
 373
 374         case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
 375                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
 376
 377                 KASSERT(txq != NULL, ("txq == NULL"));
 378                 KASSERT(evq->index == txq->evq_index,
 379                     ("evq->index != txq->evq_index"));
 380
 381                 sfxge_tx_qflush_done(txq);
 382                 break;
 383         }
 384         default:
 385                 break;
 386         }
 387
 388         return (B_FALSE);
 389 }
 390
 391 static boolean_t
 392 sfxge_ev_sram(void *arg, uint32_t code)
 393 {
 394         (void)arg;
 395         (void)code;
 396
 397         switch (code) {
 398         case EFX_SRAM_UPDATE:
 399                 EFSYS_PROBE(sram_update);
 400                 break;
 401
 402         case EFX_SRAM_CLEAR:
 403                 EFSYS_PROBE(sram_clear);
 404                 break;
 405
 406         case EFX_SRAM_ILLEGAL_CLEAR:
 407                 EFSYS_PROBE(sram_illegal_clear);
 408                 break;
 409
 410         default:
 411                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 412                 break;
 413         }
 414
 415         return (B_FALSE);
 416 }
 417
 418 static boolean_t
 419 sfxge_ev_timer(void *arg, uint32_t index)
 420 {
 421         (void)arg;
 422         (void)index;
 423
 424         return (B_FALSE);
 425 }
 426
 427 static boolean_t
 428 sfxge_ev_wake_up(void *arg, uint32_t index)
 429 {
 430         (void)arg;
 431         (void)index;
 432
 433         return (B_FALSE);
 434 }
 435
 436 #if EFSYS_OPT_QSTATS
 437
 438 static void
 439 sfxge_ev_stat_update(struct sfxge_softc *sc)
 440 {
 441         struct sfxge_evq *evq;
 442         unsigned int index;
 443         clock_t now;
 444
 445         SFXGE_ADAPTER_LOCK(sc);
 446
 447         if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
 448                 goto out;
 449
 450         now = ticks;
 451         if (now - sc->ev_stats_update_time < hz)
 452                 goto out;
 453
 454         sc->ev_stats_update_time = now;
 455
 456         /* Add event counts from each event queue in turn */
 457         for (index = 0; index < sc->evq_count; index++) {
 458                 evq = sc->evq[index];
 459                 SFXGE_EVQ_LOCK(evq);
 460                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 461                 SFXGE_EVQ_UNLOCK(evq);
 462         }
 463 out:
 464         SFXGE_ADAPTER_UNLOCK(sc);
 465 }
 466
 467 static int
 468 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 469 {
 470         struct sfxge_softc *sc = arg1;
 471         unsigned int id = arg2;
 472
 473         sfxge_ev_stat_update(sc);
 474
 475         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
 476 }
 477
 478 static void
 479 sfxge_ev_stat_init(struct sfxge_softc *sc)
 480 {
 481         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 482         struct sysctl_oid_list *stat_list;
 483         unsigned int id;
 484         char name[40];
 485
 486         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 487
 488         for (id = 0; id < EV_NQSTATS; id++) {
 489                 snprintf(name, sizeof(name), "ev_%s",
 490                          efx_ev_qstat_name(sc->enp, id));
 491                 SYSCTL_ADD_PROC(
 492                         ctx, stat_list,
 493                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 494                         sc, id, sfxge_ev_stat_handler, "Q",
 495                         "");
 496         }
 497 }
 498
 499 #endif /* EFSYS_OPT_QSTATS */
 500
 501 static void
 502 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 503 {
 504         struct sfxge_evq *evq;
 505         efx_evq_t *eep;
 506
 507         evq = sc->evq[idx];
 508         eep = evq->common;
 509
 510         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 511             ("evq->init_state != SFXGE_EVQ_STARTED"));
 512
 513         (void)efx_ev_qmoderate(eep, us);
 514 }
 515
 516 static int
 517 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 518 {
 519         struct sfxge_softc *sc = arg1;
 520         struct sfxge_intr *intr = &sc->intr;
 521         unsigned int moderation;
 522         int error;
 523         unsigned int index;
 524
 525         SFXGE_ADAPTER_LOCK(sc);
 526
 527         if (req->newptr != NULL) {
 528                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 529                     != 0)
 530                         goto out;
 531
 532                 /* We may not be calling efx_ev_qmoderate() now,
 533                  * so we have to range-check the value ourselves.
 534                  */
 535                 if (moderation >
 536                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
 537                         error = EINVAL;
 538                         goto out;
 539                 }
 540
 541                 sc->ev_moderation = moderation;
 542                 if (intr->state == SFXGE_INTR_STARTED) {
 543                         for (index = 0; index < sc->evq_count; index++)
 544                                 sfxge_ev_qmoderate(sc, index, moderation);
 545                 }
 546         } else {
 547                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 548                                    sizeof(sc->ev_moderation));
 549         }
 550
 551 out:
 552         SFXGE_ADAPTER_UNLOCK(sc);
 553
 554         return (error);
 555 }
 556
 557 static boolean_t
 558 sfxge_ev_initialized(void *arg)
 559 {
 560         struct sfxge_evq *evq;
 561
 562         evq = (struct sfxge_evq *)arg;
 563         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 564
 565         /* Init done events may be duplicated on 7xxx */
 566         KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
 567                 evq->init_state == SFXGE_EVQ_STARTED,
 568             ("evq not starting"));
 569
 570         evq->init_state = SFXGE_EVQ_STARTED;
 571
 572         return (0);
 573 }
 574
 575 static boolean_t
 576 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 577 {
 578         struct sfxge_evq *evq;
 579         struct sfxge_softc *sc;
 580
 581         evq = (struct sfxge_evq *)arg;
 582         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 583
 584         sc = evq->sc;
 585
 586         sfxge_mac_link_update(sc, link_mode);
 587
 588         return (0);
 589 }
 590
 591 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 592         .eec_initialized        = sfxge_ev_initialized,
 593         .eec_rx                 = sfxge_ev_rx,
 594         .eec_tx                 = sfxge_ev_tx,
 595         .eec_exception          = sfxge_ev_exception,
 596         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 597         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 598         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 599         .eec_software           = sfxge_ev_software,
 600         .eec_sram               = sfxge_ev_sram,
 601         .eec_wake_up            = sfxge_ev_wake_up,
 602         .eec_timer              = sfxge_ev_timer,
 603         .eec_link_change        = sfxge_ev_link_change,
 604 };
 605
 606
 607 int
 608 sfxge_ev_qpoll(struct sfxge_evq *evq)
 609 {
 610         int rc;
 611
 612         SFXGE_EVQ_LOCK(evq);
 613
 614         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
 615                             evq->init_state != SFXGE_EVQ_STARTED)) {
 616                 rc = EINVAL;
 617                 goto fail;
 618         }
 619
 620         /* Synchronize the DMA memory for reading */
 621         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 622             BUS_DMASYNC_POSTREAD);
 623
 624         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 625         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 626         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 627         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 628
 629         /* Poll the queue */
 630         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 631
 632         evq->rx_done = 0;
 633         evq->tx_done = 0;
 634
 635         /* Perform any pending completion processing */
 636         sfxge_ev_qcomplete(evq, B_TRUE);
 637
 638         /* Re-prime the event queue for interrupts */
 639         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 640                 goto fail;
 641
 642         SFXGE_EVQ_UNLOCK(evq);
 643
 644         return (0);
 645
 646 fail:
 647         SFXGE_EVQ_UNLOCK(evq);
 648         return (rc);
 649 }
 650
 651 static void
 652 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 653 {
 654         struct sfxge_evq *evq;
 655
 656         evq = sc->evq[index];
 657
 658         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 659             ("evq->init_state != SFXGE_EVQ_STARTED"));
 660
 661         SFXGE_EVQ_LOCK(evq);
 662         evq->init_state = SFXGE_EVQ_INITIALIZED;
 663         evq->read_ptr = 0;
 664         evq->exception = B_FALSE;
 665
 666 #if EFSYS_OPT_QSTATS
 667         /* Add event counts before discarding the common evq state */
 668         efx_ev_qstats_update(evq->common, sc->ev_stats);
 669 #endif
 670
 671         efx_ev_qdestroy(evq->common);
 672         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 673             EFX_EVQ_NBUFS(evq->entries));
 674         SFXGE_EVQ_UNLOCK(evq);
 675 }
 676
 677 static int
 678 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 679 {
 680         struct sfxge_evq *evq;
 681         efsys_mem_t *esmp;
 682         int count;
 683         int rc;
 684
 685         evq = sc->evq[index];
 686         esmp = &evq->mem;
 687
 688         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 689             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 690
 691         /* Clear all events. */
 692         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
 693
 694         /* Program the buffer table. */
 695         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 696             EFX_EVQ_NBUFS(evq->entries))) != 0)
 697                 return (rc);
 698
 699         /* Create the common code event queue. */
 700         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
 701             evq->buf_base_id, sc->ev_moderation, &evq->common)) != 0)
 702                 goto fail;
 703
 704         SFXGE_EVQ_LOCK(evq);
 705
 706         /* Prime the event queue for interrupts */
 707         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 708                 goto fail2;
 709
 710         evq->init_state = SFXGE_EVQ_STARTING;
 711
 712         SFXGE_EVQ_UNLOCK(evq);
 713
 714         /* Wait for the initialization event */
 715         count = 0;
 716         do {
 717                 /* Pause for 100 ms */
 718                 pause("sfxge evq init", hz / 10);
 719
 720                 /* Check to see if the test event has been processed */
 721                 if (evq->init_state == SFXGE_EVQ_STARTED)
 722                         goto done;
 723
 724         } while (++count < 20);
 725
 726         rc = ETIMEDOUT;
 727         goto fail3;
 728
 729 done:
 730         return (0);
 731
 732 fail3:
 733         SFXGE_EVQ_LOCK(evq);
 734         evq->init_state = SFXGE_EVQ_INITIALIZED;
 735 fail2:
 736         SFXGE_EVQ_UNLOCK(evq);
 737         efx_ev_qdestroy(evq->common);
 738 fail:
 739         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 740             EFX_EVQ_NBUFS(evq->entries));
 741
 742         return (rc);
 743 }
 744
 745 void
 746 sfxge_ev_stop(struct sfxge_softc *sc)
 747 {
 748         struct sfxge_intr *intr;
 749         efx_nic_t *enp;
 750         int index;
 751
 752         intr = &sc->intr;
 753         enp = sc->enp;
 754
 755         KASSERT(intr->state == SFXGE_INTR_STARTED,
 756             ("Interrupts not started"));
 757
 758         /* Stop the event queue(s) */
 759         index = sc->evq_count;
 760         while (--index >= 0)
 761                 sfxge_ev_qstop(sc, index);
 762
 763         /* Tear down the event module */
 764         efx_ev_fini(enp);
 765 }
 766
 767 int
 768 sfxge_ev_start(struct sfxge_softc *sc)
 769 {
 770         struct sfxge_intr *intr;
 771         int index;
 772         int rc;
 773
 774         intr = &sc->intr;
 775
 776         KASSERT(intr->state == SFXGE_INTR_STARTED,
 777             ("intr->state != SFXGE_INTR_STARTED"));
 778
 779         /* Initialize the event module */
 780         if ((rc = efx_ev_init(sc->enp)) != 0)
 781                 return (rc);
 782
 783         /* Start the event queues */
 784         for (index = 0; index < sc->evq_count; index++) {
 785                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 786                         goto fail;
 787         }
 788
 789         return (0);
 790
 791 fail:
 792         /* Stop the event queue(s) */
 793         while (--index >= 0)
 794                 sfxge_ev_qstop(sc, index);
 795
 796         /* Tear down the event module */
 797         efx_ev_fini(sc->enp);
 798
 799         return (rc);
 800 }
 801
 802 static void
 803 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 804 {
 805         struct sfxge_evq *evq;
 806
 807         evq = sc->evq[index];
 808
 809         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 810             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 811         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 812
 813         sfxge_dma_free(&evq->mem);
 814
 815         sc->evq[index] = NULL;
 816
 817         SFXGE_EVQ_LOCK_DESTROY(evq);
 818
 819         free(evq, M_SFXGE);
 820 }
 821
 822 static int
 823 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 824 {
 825         struct sfxge_evq *evq;
 826         efsys_mem_t *esmp;
 827         int rc;
 828
 829         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 830
 831         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 832         evq->sc = sc;
 833         evq->index = index;
 834         sc->evq[index] = evq;
 835         esmp = &evq->mem;
 836
 837         /* Build an event queue with room for one event per tx and rx buffer,
 838          * plus some extra for link state events and MCDI completions.
 839          * There are three tx queues in the first event queue and one in
 840          * other.
 841          */
 842         if (index == 0)
 843                 evq->entries =
 844                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 845                                            3 * sc->txq_entries +
 846                                            128);
 847         else
 848                 evq->entries =
 849                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 850                                            sc->txq_entries +
 851                                            128);
 852
 853         /* Initialise TX completion list */
 854         evq->txqs = &evq->txq;
 855
 856         /* Allocate DMA space. */
 857         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
 858                 return (rc);
 859
 860         /* Allocate buffer table entries. */
 861         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
 862                                  &evq->buf_base_id);
 863
 864         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
 865
 866         evq->init_state = SFXGE_EVQ_INITIALIZED;
 867
 868         return (0);
 869 }
 870
 871 void
 872 sfxge_ev_fini(struct sfxge_softc *sc)
 873 {
 874         struct sfxge_intr *intr;
 875         int index;
 876
 877         intr = &sc->intr;
 878
 879         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 880             ("intr->state != SFXGE_INTR_INITIALIZED"));
 881
 882         sc->ev_moderation = 0;
 883
 884         /* Tear down the event queue(s). */
 885         index = sc->evq_count;
 886         while (--index >= 0)
 887                 sfxge_ev_qfini(sc, index);
 888
 889         sc->evq_count = 0;
 890 }
 891
 892 int
 893 sfxge_ev_init(struct sfxge_softc *sc)
 894 {
 895         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 896         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 897         struct sfxge_intr *intr;
 898         int index;
 899         int rc;
 900
 901         intr = &sc->intr;
 902
 903         sc->evq_count = intr->n_alloc;
 904
 905         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 906             ("intr->state != SFXGE_INTR_INITIALIZED"));
 907
 908         /* Set default interrupt moderation; add a sysctl to
 909          * read and change it.
 910          */
 911         sc->ev_moderation = SFXGE_MODERATION;
 912         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 913                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 914                         sc, 0, sfxge_int_mod_handler, "IU",
 915                         "sfxge interrupt moderation (us)");
 916
 917         /*
 918          * Initialize the event queue(s) - one per interrupt.
 919          */
 920         for (index = 0; index < sc->evq_count; index++) {
 921                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 922                         goto fail;
 923         }
 924
 925 #if EFSYS_OPT_QSTATS
 926         sfxge_ev_stat_init(sc);
 927 #endif
 928
 929         return (0);
 930
 931 fail:
 932         while (--index >= 0)
 933                 sfxge_ev_qfini(sc, index);
 934
 935         sc->evq_count = 0;
 936         return (rc);
 937 }