sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/kernel.h>
  35 #include <sys/malloc.h>
  36 #include <sys/param.h>
  37 #include <sys/queue.h>
  38 #include <sys/systm.h>
  39 #include <sys/taskqueue.h>
  40
  41 #include "common/efx.h"
  42
  43 #include "sfxge.h"
  44
  45 static void
  46 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  47 {
  48         struct sfxge_softc *sc;
  49         unsigned int index;
  50         struct sfxge_rxq *rxq;
  51         struct sfxge_txq *txq;
  52
  53         sc = evq->sc;
  54         index = evq->index;
  55         rxq = sc->rxq[index];
  56
  57         if ((txq = evq->txq) != NULL) {
  58                 evq->txq = NULL;
  59                 evq->txqs = &(evq->txq);
  60
  61                 do {
  62                         struct sfxge_txq *next;
  63
  64                         next = txq->next;
  65                         txq->next = NULL;
  66
  67                         KASSERT(txq->evq_index == index,
  68                             ("txq->evq_index != index"));
  69
  70                         if (txq->pending != txq->completed)
  71                                 sfxge_tx_qcomplete(txq, evq);
  72
  73                         txq = next;
  74                 } while (txq != NULL);
  75         }
  76
  77         if (rxq->pending != rxq->completed)
  78                 sfxge_rx_qcomplete(rxq, eop);
  79 }
  80
  81 static boolean_t
  82 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  83     uint16_t flags)
  84 {
  85         struct sfxge_evq *evq;
  86         struct sfxge_softc *sc;
  87         struct sfxge_rxq *rxq;
  88         unsigned int expected;
  89         struct sfxge_rx_sw_desc *rx_desc;
  90
  91         evq = arg;
  92         sc = evq->sc;
  93
  94         if (evq->exception)
  95                 goto done;
  96
  97         rxq = sc->rxq[label];
  98         KASSERT(rxq != NULL, ("rxq == NULL"));
  99         KASSERT(evq->index == rxq->index,
 100             ("evq->index != rxq->index"));
 101
 102         if (rxq->init_state != SFXGE_RXQ_STARTED)
 103                 goto done;
 104
 105         expected = rxq->pending++ & rxq->ptr_mask;
 106         if (id != expected) {
 107                 evq->exception = B_TRUE;
 108
 109                 device_printf(sc->dev, "RX completion out of order"
 110                               " (id=%#x expected=%#x flags=%#x); resetting\n",
 111                               id, expected, flags);
 112                 sfxge_schedule_reset(sc);
 113
 114                 goto done;
 115         }
 116
 117         rx_desc = &rxq->queue[id];
 118
 119         KASSERT(rx_desc->flags == EFX_DISCARD,
 120             ("rx_desc->flags != EFX_DISCARD"));
 121         rx_desc->flags = flags;
 122
 123         KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 124         rx_desc->size = (uint16_t)size;
 125         prefetch_read_many(rx_desc->mbuf);
 126
 127         evq->rx_done++;
 128
 129         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 130                 sfxge_ev_qcomplete(evq, B_FALSE);
 131
 132 done:
 133         return (evq->rx_done >= SFXGE_EV_BATCH);
 134 }
 135
 136 static boolean_t
 137 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 138 {
 139         struct sfxge_evq *evq;
 140         struct sfxge_softc *sc;
 141
 142         evq = (struct sfxge_evq *)arg;
 143         sc = evq->sc;
 144
 145         evq->exception = B_TRUE;
 146
 147         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 148                 device_printf(sc->dev,
 149                               "hardware exception (code=%u); resetting\n",
 150                               code);
 151                 sfxge_schedule_reset(sc);
 152         }
 153
 154         return (B_FALSE);
 155 }
 156
 157 static boolean_t
 158 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
 159 {
 160         struct sfxge_evq *evq;
 161         struct sfxge_softc *sc;
 162         struct sfxge_rxq *rxq;
 163         unsigned int index;
 164         unsigned int label;
 165         uint16_t magic;
 166
 167         evq = (struct sfxge_evq *)arg;
 168         sc = evq->sc;
 169         rxq = sc->rxq[rxq_index];
 170
 171         KASSERT(rxq != NULL, ("rxq == NULL"));
 172
 173         /* Resend a software event on the correct queue */
 174         index = rxq->index;
 175         evq = sc->evq[index];
 176
 177         label = rxq_index;
 178         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 179             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
 180         magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
 181
 182         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 183             ("evq not started"));
 184         efx_ev_qpost(evq->common, magic);
 185
 186         return (B_FALSE);
 187 }
 188
 189 static boolean_t
 190 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
 191 {
 192         struct sfxge_evq *evq;
 193         struct sfxge_softc *sc;
 194         struct sfxge_rxq *rxq;
 195         unsigned int index;
 196         unsigned int label;
 197         uint16_t magic;
 198
 199         evq = (struct sfxge_evq *)arg;
 200         sc = evq->sc;
 201         rxq = sc->rxq[rxq_index];
 202
 203         KASSERT(rxq != NULL, ("rxq == NULL"));
 204
 205         /* Resend a software event on the correct queue */
 206         index = rxq->index;
 207         evq = sc->evq[index];
 208
 209         label = rxq_index;
 210         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 211             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 212         magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
 213
 214         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 215             ("evq not started"));
 216         efx_ev_qpost(evq->common, magic);
 217
 218         return (B_FALSE);
 219 }
 220
 221 static struct sfxge_txq *
 222 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
 223 {
 224         unsigned int index;
 225
 226         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
 227             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
 228         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
 229         return (evq->sc->txq[index]);
 230 }
 231
 232 static boolean_t
 233 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 234 {
 235         struct sfxge_evq *evq;
 236         struct sfxge_txq *txq;
 237         unsigned int stop;
 238         unsigned int delta;
 239
 240         evq = (struct sfxge_evq *)arg;
 241         txq = sfxge_get_txq_by_label(evq, label);
 242
 243         KASSERT(txq != NULL, ("txq == NULL"));
 244         KASSERT(evq->index == txq->evq_index,
 245             ("evq->index != txq->evq_index"));
 246
 247         if (txq->init_state != SFXGE_TXQ_STARTED)
 248                 goto done;
 249
 250         stop = (id + 1) & txq->ptr_mask;
 251         id = txq->pending & txq->ptr_mask;
 252
 253         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
 254         txq->pending += delta;
 255
 256         evq->tx_done++;
 257
 258         if (txq->next == NULL &&
 259             evq->txqs != &(txq->next)) {
 260                 *(evq->txqs) = txq;
 261                 evq->txqs = &(txq->next);
 262         }
 263
 264         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 265                 sfxge_tx_qcomplete(txq, evq);
 266
 267 done:
 268         return (evq->tx_done >= SFXGE_EV_BATCH);
 269 }
 270
 271 static boolean_t
 272 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
 273 {
 274         struct sfxge_evq *evq;
 275         struct sfxge_softc *sc;
 276         struct sfxge_txq *txq;
 277         unsigned int label;
 278         uint16_t magic;
 279
 280         evq = (struct sfxge_evq *)arg;
 281         sc = evq->sc;
 282         txq = sc->txq[txq_index];
 283
 284         KASSERT(txq != NULL, ("txq == NULL"));
 285         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 286             ("txq not initialized"));
 287
 288         /* Resend a software event on the correct queue */
 289         evq = sc->evq[txq->evq_index];
 290
 291         label = txq->type;
 292         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 293             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 294         magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
 295
 296         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 297             ("evq not started"));
 298         efx_ev_qpost(evq->common, magic);
 299
 300         return (B_FALSE);
 301 }
 302
 303 static boolean_t
 304 sfxge_ev_software(void *arg, uint16_t magic)
 305 {
 306         struct sfxge_evq *evq;
 307         struct sfxge_softc *sc;
 308         unsigned int label;
 309
 310         evq = (struct sfxge_evq *)arg;
 311         sc = evq->sc;
 312
 313         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 314         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 315
 316         switch (magic) {
 317         case SFXGE_MAGIC_RX_QFLUSH_DONE: {
 318                 struct sfxge_rxq *rxq = sc->rxq[label];
 319
 320                 KASSERT(rxq != NULL, ("rxq == NULL"));
 321                 KASSERT(evq->index == rxq->index,
 322                     ("evq->index != rxq->index"));
 323
 324                 sfxge_rx_qflush_done(rxq);
 325                 break;
 326         }
 327         case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
 328                 struct sfxge_rxq *rxq = sc->rxq[label];
 329
 330                 KASSERT(rxq != NULL, ("rxq == NULL"));
 331                 KASSERT(evq->index == rxq->index,
 332                     ("evq->index != rxq->index"));
 333
 334                 sfxge_rx_qflush_failed(rxq);
 335                 break;
 336         }
 337         case SFXGE_MAGIC_RX_QREFILL: {
 338                 struct sfxge_rxq *rxq = sc->rxq[label];
 339
 340                 KASSERT(rxq != NULL, ("rxq == NULL"));
 341                 KASSERT(evq->index == rxq->index,
 342                     ("evq->index != rxq->index"));
 343
 344                 sfxge_rx_qrefill(rxq);
 345                 break;
 346         }
 347         case SFXGE_MAGIC_TX_QFLUSH_DONE: {
 348                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
 349
 350                 KASSERT(txq != NULL, ("txq == NULL"));
 351                 KASSERT(evq->index == txq->evq_index,
 352                     ("evq->index != txq->evq_index"));
 353
 354                 sfxge_tx_qflush_done(txq);
 355                 break;
 356         }
 357         default:
 358                 break;
 359         }
 360
 361         return (B_FALSE);
 362 }
 363
 364 static boolean_t
 365 sfxge_ev_sram(void *arg, uint32_t code)
 366 {
 367         (void)arg;
 368         (void)code;
 369
 370         switch (code) {
 371         case EFX_SRAM_UPDATE:
 372                 EFSYS_PROBE(sram_update);
 373                 break;
 374
 375         case EFX_SRAM_CLEAR:
 376                 EFSYS_PROBE(sram_clear);
 377                 break;
 378
 379         case EFX_SRAM_ILLEGAL_CLEAR:
 380                 EFSYS_PROBE(sram_illegal_clear);
 381                 break;
 382
 383         default:
 384                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 385                 break;
 386         }
 387
 388         return (B_FALSE);
 389 }
 390
 391 static boolean_t
 392 sfxge_ev_timer(void *arg, uint32_t index)
 393 {
 394         (void)arg;
 395         (void)index;
 396
 397         return (B_FALSE);
 398 }
 399
 400 static boolean_t
 401 sfxge_ev_wake_up(void *arg, uint32_t index)
 402 {
 403         (void)arg;
 404         (void)index;
 405
 406         return (B_FALSE);
 407 }
 408
 409 #if EFSYS_OPT_QSTATS
 410
 411 static void
 412 sfxge_ev_stat_update(struct sfxge_softc *sc)
 413 {
 414         struct sfxge_evq *evq;
 415         unsigned int index;
 416         clock_t now;
 417
 418         SFXGE_ADAPTER_LOCK(sc);
 419
 420         if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
 421                 goto out;
 422
 423         now = ticks;
 424         if (now - sc->ev_stats_update_time < hz)
 425                 goto out;
 426
 427         sc->ev_stats_update_time = now;
 428
 429         /* Add event counts from each event queue in turn */
 430         for (index = 0; index < sc->evq_count; index++) {
 431                 evq = sc->evq[index];
 432                 SFXGE_EVQ_LOCK(evq);
 433                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 434                 SFXGE_EVQ_UNLOCK(evq);
 435         }
 436 out:
 437         SFXGE_ADAPTER_UNLOCK(sc);
 438 }
 439
 440 static int
 441 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 442 {
 443         struct sfxge_softc *sc = arg1;
 444         unsigned int id = arg2;
 445
 446         sfxge_ev_stat_update(sc);
 447
 448         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
 449 }
 450
 451 static void
 452 sfxge_ev_stat_init(struct sfxge_softc *sc)
 453 {
 454         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 455         struct sysctl_oid_list *stat_list;
 456         unsigned int id;
 457         char name[40];
 458
 459         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 460
 461         for (id = 0; id < EV_NQSTATS; id++) {
 462                 snprintf(name, sizeof(name), "ev_%s",
 463                          efx_ev_qstat_name(sc->enp, id));
 464                 SYSCTL_ADD_PROC(
 465                         ctx, stat_list,
 466                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 467                         sc, id, sfxge_ev_stat_handler, "Q",
 468                         "");
 469         }
 470 }
 471
 472 #endif /* EFSYS_OPT_QSTATS */
 473
 474 static void
 475 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 476 {
 477         struct sfxge_evq *evq;
 478         efx_evq_t *eep;
 479
 480         evq = sc->evq[idx];
 481         eep = evq->common;
 482
 483         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 484             ("evq->init_state != SFXGE_EVQ_STARTED"));
 485
 486         (void)efx_ev_qmoderate(eep, us);
 487 }
 488
 489 static int
 490 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 491 {
 492         struct sfxge_softc *sc = arg1;
 493         struct sfxge_intr *intr = &sc->intr;
 494         unsigned int moderation;
 495         int error;
 496         unsigned int index;
 497
 498         SFXGE_ADAPTER_LOCK(sc);
 499
 500         if (req->newptr != NULL) {
 501                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 502                     != 0)
 503                         goto out;
 504
 505                 /* We may not be calling efx_ev_qmoderate() now,
 506                  * so we have to range-check the value ourselves.
 507                  */
 508                 if (moderation >
 509                     efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
 510                         error = EINVAL;
 511                         goto out;
 512                 }
 513
 514                 sc->ev_moderation = moderation;
 515                 if (intr->state == SFXGE_INTR_STARTED) {
 516                         for (index = 0; index < sc->evq_count; index++)
 517                                 sfxge_ev_qmoderate(sc, index, moderation);
 518                 }
 519         } else {
 520                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 521                                    sizeof(sc->ev_moderation));
 522         }
 523
 524 out:
 525         SFXGE_ADAPTER_UNLOCK(sc);
 526
 527         return (error);
 528 }
 529
 530 static boolean_t
 531 sfxge_ev_initialized(void *arg)
 532 {
 533         struct sfxge_evq *evq;
 534
 535         evq = (struct sfxge_evq *)arg;
 536
 537         KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
 538             ("evq not starting"));
 539
 540         evq->init_state = SFXGE_EVQ_STARTED;
 541
 542         return (0);
 543 }
 544
 545 static boolean_t
 546 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 547 {
 548         struct sfxge_evq *evq;
 549         struct sfxge_softc *sc;
 550
 551         evq = (struct sfxge_evq *)arg;
 552         sc = evq->sc;
 553
 554         sfxge_mac_link_update(sc, link_mode);
 555
 556         return (0);
 557 }
 558
 559 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 560         .eec_initialized        = sfxge_ev_initialized,
 561         .eec_rx                 = sfxge_ev_rx,
 562         .eec_tx                 = sfxge_ev_tx,
 563         .eec_exception          = sfxge_ev_exception,
 564         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 565         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 566         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 567         .eec_software           = sfxge_ev_software,
 568         .eec_sram               = sfxge_ev_sram,
 569         .eec_wake_up            = sfxge_ev_wake_up,
 570         .eec_timer              = sfxge_ev_timer,
 571         .eec_link_change        = sfxge_ev_link_change,
 572 };
 573
 574
 575 int
 576 sfxge_ev_qpoll(struct sfxge_evq *evq)
 577 {
 578         int rc;
 579
 580         SFXGE_EVQ_LOCK(evq);
 581
 582         if (evq->init_state != SFXGE_EVQ_STARTING &&
 583             evq->init_state != SFXGE_EVQ_STARTED) {
 584                 rc = EINVAL;
 585                 goto fail;
 586         }
 587
 588         /* Synchronize the DMA memory for reading */
 589         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 590             BUS_DMASYNC_POSTREAD);
 591
 592         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 593         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 594         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 595         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 596
 597         /* Poll the queue */
 598         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 599
 600         evq->rx_done = 0;
 601         evq->tx_done = 0;
 602
 603         /* Perform any pending completion processing */
 604         sfxge_ev_qcomplete(evq, B_TRUE);
 605
 606         /* Re-prime the event queue for interrupts */
 607         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 608                 goto fail;
 609
 610         SFXGE_EVQ_UNLOCK(evq);
 611
 612         return (0);
 613
 614 fail:
 615         SFXGE_EVQ_UNLOCK(evq);
 616         return (rc);
 617 }
 618
 619 static void
 620 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 621 {
 622         struct sfxge_evq *evq;
 623
 624         evq = sc->evq[index];
 625
 626         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 627             ("evq->init_state != SFXGE_EVQ_STARTED"));
 628
 629         SFXGE_EVQ_LOCK(evq);
 630         evq->init_state = SFXGE_EVQ_INITIALIZED;
 631         evq->read_ptr = 0;
 632         evq->exception = B_FALSE;
 633
 634 #if EFSYS_OPT_QSTATS
 635         /* Add event counts before discarding the common evq state */
 636         efx_ev_qstats_update(evq->common, sc->ev_stats);
 637 #endif
 638
 639         efx_ev_qdestroy(evq->common);
 640         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 641             EFX_EVQ_NBUFS(evq->entries));
 642         SFXGE_EVQ_UNLOCK(evq);
 643 }
 644
 645 static int
 646 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 647 {
 648         struct sfxge_evq *evq;
 649         efsys_mem_t *esmp;
 650         int count;
 651         int rc;
 652
 653         evq = sc->evq[index];
 654         esmp = &evq->mem;
 655
 656         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 657             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 658
 659         /* Clear all events. */
 660         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
 661
 662         /* Program the buffer table. */
 663         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 664             EFX_EVQ_NBUFS(evq->entries))) != 0)
 665                 return (rc);
 666
 667         /* Create the common code event queue. */
 668         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
 669             evq->buf_base_id, &evq->common)) != 0)
 670                 goto fail;
 671
 672         SFXGE_EVQ_LOCK(evq);
 673
 674         /* Set the default moderation */
 675         (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
 676
 677         /* Prime the event queue for interrupts */
 678         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 679                 goto fail2;
 680
 681         evq->init_state = SFXGE_EVQ_STARTING;
 682
 683         SFXGE_EVQ_UNLOCK(evq);
 684
 685         /* Wait for the initialization event */
 686         count = 0;
 687         do {
 688                 /* Pause for 100 ms */
 689                 pause("sfxge evq init", hz / 10);
 690
 691                 /* Check to see if the test event has been processed */
 692                 if (evq->init_state == SFXGE_EVQ_STARTED)
 693                         goto done;
 694
 695         } while (++count < 20);
 696
 697         rc = ETIMEDOUT;
 698         goto fail3;
 699
 700 done:
 701         return (0);
 702
 703 fail3:
 704         SFXGE_EVQ_LOCK(evq);
 705         evq->init_state = SFXGE_EVQ_INITIALIZED;
 706 fail2:
 707         SFXGE_EVQ_UNLOCK(evq);
 708         efx_ev_qdestroy(evq->common);
 709 fail:
 710         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 711             EFX_EVQ_NBUFS(evq->entries));
 712
 713         return (rc);
 714 }
 715
 716 void
 717 sfxge_ev_stop(struct sfxge_softc *sc)
 718 {
 719         struct sfxge_intr *intr;
 720         efx_nic_t *enp;
 721         int index;
 722
 723         intr = &sc->intr;
 724         enp = sc->enp;
 725
 726         KASSERT(intr->state == SFXGE_INTR_STARTED,
 727             ("Interrupts not started"));
 728
 729         /* Stop the event queue(s) */
 730         index = sc->evq_count;
 731         while (--index >= 0)
 732                 sfxge_ev_qstop(sc, index);
 733
 734         /* Tear down the event module */
 735         efx_ev_fini(enp);
 736 }
 737
 738 int
 739 sfxge_ev_start(struct sfxge_softc *sc)
 740 {
 741         struct sfxge_intr *intr;
 742         int index;
 743         int rc;
 744
 745         intr = &sc->intr;
 746
 747         KASSERT(intr->state == SFXGE_INTR_STARTED,
 748             ("intr->state != SFXGE_INTR_STARTED"));
 749
 750         /* Initialize the event module */
 751         if ((rc = efx_ev_init(sc->enp)) != 0)
 752                 return (rc);
 753
 754         /* Start the event queues */
 755         for (index = 0; index < sc->evq_count; index++) {
 756                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 757                         goto fail;
 758         }
 759
 760         return (0);
 761
 762 fail:
 763         /* Stop the event queue(s) */
 764         while (--index >= 0)
 765                 sfxge_ev_qstop(sc, index);
 766
 767         /* Tear down the event module */
 768         efx_ev_fini(sc->enp);
 769
 770         return (rc);
 771 }
 772
 773 static void
 774 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 775 {
 776         struct sfxge_evq *evq;
 777
 778         evq = sc->evq[index];
 779
 780         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 781             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 782         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 783
 784         sfxge_dma_free(&evq->mem);
 785
 786         sc->evq[index] = NULL;
 787
 788         SFXGE_EVQ_LOCK_DESTROY(evq);
 789
 790         free(evq, M_SFXGE);
 791 }
 792
 793 static int
 794 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 795 {
 796         struct sfxge_evq *evq;
 797         efsys_mem_t *esmp;
 798         int rc;
 799
 800         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 801
 802         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 803         evq->sc = sc;
 804         evq->index = index;
 805         sc->evq[index] = evq;
 806         esmp = &evq->mem;
 807
 808         /* Build an event queue with room for one event per tx and rx buffer,
 809          * plus some extra for link state events and MCDI completions.
 810          * There are three tx queues in the first event queue and one in
 811          * other.
 812          */
 813         if (index == 0)
 814                 evq->entries =
 815                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 816                                            3 * sc->txq_entries +
 817                                            128);
 818         else
 819                 evq->entries =
 820                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 821                                            sc->txq_entries +
 822                                            128);
 823
 824         /* Initialise TX completion list */
 825         evq->txqs = &evq->txq;
 826
 827         /* Allocate DMA space. */
 828         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
 829                 return (rc);
 830
 831         /* Allocate buffer table entries. */
 832         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
 833                                  &evq->buf_base_id);
 834
 835         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
 836
 837         evq->init_state = SFXGE_EVQ_INITIALIZED;
 838
 839         return (0);
 840 }
 841
 842 void
 843 sfxge_ev_fini(struct sfxge_softc *sc)
 844 {
 845         struct sfxge_intr *intr;
 846         int index;
 847
 848         intr = &sc->intr;
 849
 850         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 851             ("intr->state != SFXGE_INTR_INITIALIZED"));
 852
 853         sc->ev_moderation = 0;
 854
 855         /* Tear down the event queue(s). */
 856         index = sc->evq_count;
 857         while (--index >= 0)
 858                 sfxge_ev_qfini(sc, index);
 859
 860         sc->evq_count = 0;
 861 }
 862
 863 int
 864 sfxge_ev_init(struct sfxge_softc *sc)
 865 {
 866         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 867         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 868         struct sfxge_intr *intr;
 869         int index;
 870         int rc;
 871
 872         intr = &sc->intr;
 873
 874         sc->evq_count = intr->n_alloc;
 875
 876         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 877             ("intr->state != SFXGE_INTR_INITIALIZED"));
 878
 879         /* Set default interrupt moderation; add a sysctl to
 880          * read and change it.
 881          */
 882         sc->ev_moderation = SFXGE_MODERATION;
 883         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 884                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 885                         sc, 0, sfxge_int_mod_handler, "IU",
 886                         "sfxge interrupt moderation (us)");
 887
 888         /*
 889          * Initialize the event queue(s) - one per interrupt.
 890          */
 891         for (index = 0; index < sc->evq_count; index++) {
 892                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 893                         goto fail;
 894         }
 895
 896 #if EFSYS_OPT_QSTATS
 897         sfxge_ev_stat_init(sc);
 898 #endif
 899
 900         return (0);
 901
 902 fail:
 903         while (--index >= 0)
 904                 sfxge_ev_qfini(sc, index);
 905
 906         sc->evq_count = 0;
 907         return (rc);
 908 }