sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright notice,
  12  *    this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  *    this list of conditions and the following disclaimer in the documentation
  15  *    and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * The views and conclusions contained in the software and documentation are
  30  * those of the authors and should not be interpreted as representing official
  31  * policies, either expressed or implied, of the FreeBSD Project.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39
  40 #include "common/efx.h"
  41
  42 #include "sfxge.h"
  43
  44 static void
  45 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  46 {
  47         struct sfxge_softc *sc;
  48         unsigned int index;
  49         struct sfxge_rxq *rxq;
  50         struct sfxge_txq *txq;
  51
  52         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  53
  54         sc = evq->sc;
  55         index = evq->index;
  56         rxq = sc->rxq[index];
  57
  58         if ((txq = evq->txq) != NULL) {
  59                 evq->txq = NULL;
  60                 evq->txqs = &(evq->txq);
  61
  62                 do {
  63                         struct sfxge_txq *next;
  64
  65                         next = txq->next;
  66                         txq->next = NULL;
  67
  68                         KASSERT(txq->evq_index == index,
  69                             ("txq->evq_index != index"));
  70
  71                         if (txq->pending != txq->completed)
  72                                 sfxge_tx_qcomplete(txq, evq);
  73
  74                         txq = next;
  75                 } while (txq != NULL);
  76         }
  77
  78         if (rxq->pending != rxq->completed)
  79                 sfxge_rx_qcomplete(rxq, eop);
  80 }
  81
  82 static boolean_t
  83 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  84             uint16_t flags)
  85 {
  86         struct sfxge_evq *evq;
  87         struct sfxge_softc *sc;
  88         struct sfxge_rxq *rxq;
  89         unsigned int stop;
  90         unsigned int delta;
  91         struct sfxge_rx_sw_desc *rx_desc;
  92
  93         evq = arg;
  94         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  95
  96         sc = evq->sc;
  97
  98         if (evq->exception)
  99                 goto done;
 100
 101         rxq = sc->rxq[label];
 102         KASSERT(rxq != NULL, ("rxq == NULL"));
 103         KASSERT(evq->index == rxq->index,
 104             ("evq->index != rxq->index"));
 105
 106         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
 107                 goto done;
 108
 109         stop = (id + 1) & rxq->ptr_mask;
 110         id = rxq->pending & rxq->ptr_mask;
 111         delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
 112         rxq->pending += delta;
 113
 114         if (delta != 1) {
 115                 if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) ||
 116                     (delta <= 0) ||
 117                     (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
 118                         evq->exception = B_TRUE;
 119
 120                         device_printf(sc->dev, "RX completion out of order"
 121                                                   " (id=%#x delta=%u flags=%#x); resetting\n",
 122                                                   id, delta, flags);
 123                         sfxge_schedule_reset(sc);
 124
 125                         goto done;
 126                 }
 127         }
 128
 129         rx_desc = &rxq->queue[id];
 130
 131         prefetch_read_many(rx_desc->mbuf);
 132
 133         for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
 134                 rx_desc = &rxq->queue[id];
 135                 KASSERT(rx_desc->flags == EFX_DISCARD,
 136                                 ("rx_desc->flags != EFX_DISCARD"));
 137                 rx_desc->flags = flags;
 138
 139                 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 140                 rx_desc->size = (uint16_t)size;
 141         }
 142
 143         evq->rx_done++;
 144
 145         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 146                 sfxge_ev_qcomplete(evq, B_FALSE);
 147
 148 done:
 149         return (evq->rx_done >= SFXGE_EV_BATCH);
 150 }
 151
 152 static boolean_t
 153 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 154 {
 155         struct sfxge_evq *evq;
 156         struct sfxge_softc *sc;
 157
 158         evq = (struct sfxge_evq *)arg;
 159         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 160
 161         sc = evq->sc;
 162
 163         DBGPRINT(sc->dev, "[%d] %s", evq->index,
 164                           (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
 165                           (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
 166                           (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
 167                           (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
 168                           (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
 169                           (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
 170                           (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
 171                           (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
 172                           (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
 173                           "UNKNOWN");
 174
 175         evq->exception = B_TRUE;
 176
 177         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 178                 device_printf(sc->dev,
 179                               "hardware exception (code=%u); resetting\n",
 180                               code);
 181                 sfxge_schedule_reset(sc);
 182         }
 183
 184         return (B_FALSE);
 185 }
 186
 187 static boolean_t
 188 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
 189 {
 190         struct sfxge_evq *evq;
 191         struct sfxge_softc *sc;
 192         struct sfxge_rxq *rxq;
 193         unsigned int index;
 194         unsigned int label;
 195         uint16_t magic;
 196
 197         evq = (struct sfxge_evq *)arg;
 198         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 199
 200         sc = evq->sc;
 201         rxq = sc->rxq[rxq_index];
 202
 203         KASSERT(rxq != NULL, ("rxq == NULL"));
 204
 205         /* Resend a software event on the correct queue */
 206         index = rxq->index;
 207         if (index == evq->index) {
 208                 sfxge_rx_qflush_done(rxq);
 209                 return (B_FALSE);
 210         }
 211
 212         evq = sc->evq[index];
 213
 214         label = rxq_index;
 215         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 216             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
 217         magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
 218
 219         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 220             ("evq not started"));
 221         efx_ev_qpost(evq->common, magic);
 222
 223         return (B_FALSE);
 224 }
 225
 226 static boolean_t
 227 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
 228 {
 229         struct sfxge_evq *evq;
 230         struct sfxge_softc *sc;
 231         struct sfxge_rxq *rxq;
 232         unsigned int index;
 233         unsigned int label;
 234         uint16_t magic;
 235
 236         evq = (struct sfxge_evq *)arg;
 237         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 238
 239         sc = evq->sc;
 240         rxq = sc->rxq[rxq_index];
 241
 242         KASSERT(rxq != NULL, ("rxq == NULL"));
 243
 244         /* Resend a software event on the correct queue */
 245         index = rxq->index;
 246         evq = sc->evq[index];
 247
 248         label = rxq_index;
 249         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 250             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 251         magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
 252
 253         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 254             ("evq not started"));
 255         efx_ev_qpost(evq->common, magic);
 256
 257         return (B_FALSE);
 258 }
 259
 260 static struct sfxge_txq *
 261 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
 262 {
 263         unsigned int index;
 264
 265         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
 266             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
 267         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
 268         return (evq->sc->txq[index]);
 269 }
 270
 271 static boolean_t
 272 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 273 {
 274         struct sfxge_evq *evq;
 275         struct sfxge_txq *txq;
 276         unsigned int stop;
 277         unsigned int delta;
 278
 279         evq = (struct sfxge_evq *)arg;
 280         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 281
 282         txq = sfxge_get_txq_by_label(evq, label);
 283
 284         KASSERT(txq != NULL, ("txq == NULL"));
 285         KASSERT(evq->index == txq->evq_index,
 286             ("evq->index != txq->evq_index"));
 287
 288         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
 289                 goto done;
 290
 291         stop = (id + 1) & txq->ptr_mask;
 292         id = txq->pending & txq->ptr_mask;
 293
 294         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
 295         txq->pending += delta;
 296
 297         evq->tx_done++;
 298
 299         if (txq->next == NULL &&
 300             evq->txqs != &(txq->next)) {
 301                 *(evq->txqs) = txq;
 302                 evq->txqs = &(txq->next);
 303         }
 304
 305         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 306                 sfxge_tx_qcomplete(txq, evq);
 307
 308 done:
 309         return (evq->tx_done >= SFXGE_EV_BATCH);
 310 }
 311
 312 static boolean_t
 313 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
 314 {
 315         struct sfxge_evq *evq;
 316         struct sfxge_softc *sc;
 317         struct sfxge_txq *txq;
 318         unsigned int label;
 319         uint16_t magic;
 320
 321         evq = (struct sfxge_evq *)arg;
 322         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 323
 324         sc = evq->sc;
 325         txq = sc->txq[txq_index];
 326
 327         KASSERT(txq != NULL, ("txq == NULL"));
 328         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 329             ("txq not initialized"));
 330
 331         if (txq->evq_index == evq->index) {
 332                 sfxge_tx_qflush_done(txq);
 333                 return (B_FALSE);
 334         }
 335
 336         /* Resend a software event on the correct queue */
 337         evq = sc->evq[txq->evq_index];
 338
 339         label = txq->type;
 340         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 341             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 342         magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
 343
 344         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 345             ("evq not started"));
 346         efx_ev_qpost(evq->common, magic);
 347
 348         return (B_FALSE);
 349 }
 350
 351 static boolean_t
 352 sfxge_ev_software(void *arg, uint16_t magic)
 353 {
 354         struct sfxge_evq *evq;
 355         struct sfxge_softc *sc;
 356         unsigned int label;
 357
 358         evq = (struct sfxge_evq *)arg;
 359         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 360
 361         sc = evq->sc;
 362
 363         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 364         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 365
 366         switch (magic) {
 367         case SFXGE_MAGIC_RX_QFLUSH_DONE: {
 368                 struct sfxge_rxq *rxq = sc->rxq[label];
 369
 370                 KASSERT(rxq != NULL, ("rxq == NULL"));
 371                 KASSERT(evq->index == rxq->index,
 372                     ("evq->index != rxq->index"));
 373
 374                 sfxge_rx_qflush_done(rxq);
 375                 break;
 376         }
 377         case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
 378                 struct sfxge_rxq *rxq = sc->rxq[label];
 379
 380                 KASSERT(rxq != NULL, ("rxq == NULL"));
 381                 KASSERT(evq->index == rxq->index,
 382                     ("evq->index != rxq->index"));
 383
 384                 sfxge_rx_qflush_failed(rxq);
 385                 break;
 386         }
 387         case SFXGE_MAGIC_RX_QREFILL: {
 388                 struct sfxge_rxq *rxq = sc->rxq[label];
 389
 390                 KASSERT(rxq != NULL, ("rxq == NULL"));
 391                 KASSERT(evq->index == rxq->index,
 392                     ("evq->index != rxq->index"));
 393
 394                 sfxge_rx_qrefill(rxq);
 395                 break;
 396         }
 397         case SFXGE_MAGIC_TX_QFLUSH_DONE: {
 398                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
 399
 400                 KASSERT(txq != NULL, ("txq == NULL"));
 401                 KASSERT(evq->index == txq->evq_index,
 402                     ("evq->index != txq->evq_index"));
 403
 404                 sfxge_tx_qflush_done(txq);
 405                 break;
 406         }
 407         default:
 408                 break;
 409         }
 410
 411         return (B_FALSE);
 412 }
 413
 414 static boolean_t
 415 sfxge_ev_sram(void *arg, uint32_t code)
 416 {
 417         (void)arg;
 418         (void)code;
 419
 420         switch (code) {
 421         case EFX_SRAM_UPDATE:
 422                 EFSYS_PROBE(sram_update);
 423                 break;
 424
 425         case EFX_SRAM_CLEAR:
 426                 EFSYS_PROBE(sram_clear);
 427                 break;
 428
 429         case EFX_SRAM_ILLEGAL_CLEAR:
 430                 EFSYS_PROBE(sram_illegal_clear);
 431                 break;
 432
 433         default:
 434                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 435                 break;
 436         }
 437
 438         return (B_FALSE);
 439 }
 440
 441 static boolean_t
 442 sfxge_ev_timer(void *arg, uint32_t index)
 443 {
 444         (void)arg;
 445         (void)index;
 446
 447         return (B_FALSE);
 448 }
 449
 450 static boolean_t
 451 sfxge_ev_wake_up(void *arg, uint32_t index)
 452 {
 453         (void)arg;
 454         (void)index;
 455
 456         return (B_FALSE);
 457 }
 458
 459 #if EFSYS_OPT_QSTATS
 460
 461 static void
 462 sfxge_ev_stat_update(struct sfxge_softc *sc)
 463 {
 464         struct sfxge_evq *evq;
 465         unsigned int index;
 466         clock_t now;
 467
 468         SFXGE_ADAPTER_LOCK(sc);
 469
 470         if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
 471                 goto out;
 472
 473         now = ticks;
 474         if (now - sc->ev_stats_update_time < hz)
 475                 goto out;
 476
 477         sc->ev_stats_update_time = now;
 478
 479         /* Add event counts from each event queue in turn */
 480         for (index = 0; index < sc->evq_count; index++) {
 481                 evq = sc->evq[index];
 482                 SFXGE_EVQ_LOCK(evq);
 483                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 484                 SFXGE_EVQ_UNLOCK(evq);
 485         }
 486 out:
 487         SFXGE_ADAPTER_UNLOCK(sc);
 488 }
 489
 490 static int
 491 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 492 {
 493         struct sfxge_softc *sc = arg1;
 494         unsigned int id = arg2;
 495
 496         sfxge_ev_stat_update(sc);
 497
 498         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
 499 }
 500
 501 static void
 502 sfxge_ev_stat_init(struct sfxge_softc *sc)
 503 {
 504         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 505         struct sysctl_oid_list *stat_list;
 506         unsigned int id;
 507         char name[40];
 508
 509         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 510
 511         for (id = 0; id < EV_NQSTATS; id++) {
 512                 snprintf(name, sizeof(name), "ev_%s",
 513                          efx_ev_qstat_name(sc->enp, id));
 514                 SYSCTL_ADD_PROC(
 515                         ctx, stat_list,
 516                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 517                         sc, id, sfxge_ev_stat_handler, "Q",
 518                         "");
 519         }
 520 }
 521
 522 #endif /* EFSYS_OPT_QSTATS */
 523
 524 static void
 525 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 526 {
 527         struct sfxge_evq *evq;
 528         efx_evq_t *eep;
 529
 530         evq = sc->evq[idx];
 531         eep = evq->common;
 532
 533         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 534             ("evq->init_state != SFXGE_EVQ_STARTED"));
 535
 536         (void)efx_ev_qmoderate(eep, us);
 537 }
 538
 539 static int
 540 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 541 {
 542         struct sfxge_softc *sc = arg1;
 543         struct sfxge_intr *intr = &sc->intr;
 544         unsigned int moderation;
 545         int error;
 546         unsigned int index;
 547
 548         SFXGE_ADAPTER_LOCK(sc);
 549
 550         if (req->newptr != NULL) {
 551                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 552                     != 0)
 553                         goto out;
 554
 555                 /* We may not be calling efx_ev_qmoderate() now,
 556                  * so we have to range-check the value ourselves.
 557                  */
 558                 if (moderation >
 559                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
 560                         error = EINVAL;
 561                         goto out;
 562                 }
 563
 564                 sc->ev_moderation = moderation;
 565                 if (intr->state == SFXGE_INTR_STARTED) {
 566                         for (index = 0; index < sc->evq_count; index++)
 567                                 sfxge_ev_qmoderate(sc, index, moderation);
 568                 }
 569         } else {
 570                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 571                                    sizeof(sc->ev_moderation));
 572         }
 573
 574 out:
 575         SFXGE_ADAPTER_UNLOCK(sc);
 576
 577         return (error);
 578 }
 579
 580 static boolean_t
 581 sfxge_ev_initialized(void *arg)
 582 {
 583         struct sfxge_evq *evq;
 584
 585         evq = (struct sfxge_evq *)arg;
 586         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 587
 588         /* Init done events may be duplicated on 7xxx */
 589         KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
 590                 evq->init_state == SFXGE_EVQ_STARTED,
 591             ("evq not starting"));
 592
 593         evq->init_state = SFXGE_EVQ_STARTED;
 594
 595         return (0);
 596 }
 597
 598 static boolean_t
 599 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 600 {
 601         struct sfxge_evq *evq;
 602         struct sfxge_softc *sc;
 603
 604         evq = (struct sfxge_evq *)arg;
 605         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 606
 607         sc = evq->sc;
 608
 609         sfxge_mac_link_update(sc, link_mode);
 610
 611         return (0);
 612 }
 613
 614 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 615         .eec_initialized        = sfxge_ev_initialized,
 616         .eec_rx                 = sfxge_ev_rx,
 617         .eec_tx                 = sfxge_ev_tx,
 618         .eec_exception          = sfxge_ev_exception,
 619         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 620         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 621         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 622         .eec_software           = sfxge_ev_software,
 623         .eec_sram               = sfxge_ev_sram,
 624         .eec_wake_up            = sfxge_ev_wake_up,
 625         .eec_timer              = sfxge_ev_timer,
 626         .eec_link_change        = sfxge_ev_link_change,
 627 };
 628
 629
 630 int
 631 sfxge_ev_qpoll(struct sfxge_evq *evq)
 632 {
 633         int rc;
 634
 635         SFXGE_EVQ_LOCK(evq);
 636
 637         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
 638                             evq->init_state != SFXGE_EVQ_STARTED)) {
 639                 rc = EINVAL;
 640                 goto fail;
 641         }
 642
 643         /* Synchronize the DMA memory for reading */
 644         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 645             BUS_DMASYNC_POSTREAD);
 646
 647         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 648         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 649         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 650         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 651
 652         /* Poll the queue */
 653         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 654
 655         evq->rx_done = 0;
 656         evq->tx_done = 0;
 657
 658         /* Perform any pending completion processing */
 659         sfxge_ev_qcomplete(evq, B_TRUE);
 660
 661         /* Re-prime the event queue for interrupts */
 662         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 663                 goto fail;
 664
 665         SFXGE_EVQ_UNLOCK(evq);
 666
 667         return (0);
 668
 669 fail:
 670         SFXGE_EVQ_UNLOCK(evq);
 671         return (rc);
 672 }
 673
 674 static void
 675 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 676 {
 677         struct sfxge_evq *evq;
 678
 679         evq = sc->evq[index];
 680
 681         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 682             ("evq->init_state != SFXGE_EVQ_STARTED"));
 683
 684         SFXGE_EVQ_LOCK(evq);
 685         evq->init_state = SFXGE_EVQ_INITIALIZED;
 686         evq->read_ptr = 0;
 687         evq->exception = B_FALSE;
 688
 689 #if EFSYS_OPT_QSTATS
 690         /* Add event counts before discarding the common evq state */
 691         efx_ev_qstats_update(evq->common, sc->ev_stats);
 692 #endif
 693
 694         efx_ev_qdestroy(evq->common);
 695         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 696             EFX_EVQ_NBUFS(evq->entries));
 697         SFXGE_EVQ_UNLOCK(evq);
 698 }
 699
 700 static int
 701 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 702 {
 703         struct sfxge_evq *evq;
 704         efsys_mem_t *esmp;
 705         int count;
 706         int rc;
 707
 708         evq = sc->evq[index];
 709         esmp = &evq->mem;
 710
 711         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 712             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 713
 714         /* Clear all events. */
 715         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
 716
 717         /* Program the buffer table. */
 718         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 719             EFX_EVQ_NBUFS(evq->entries))) != 0)
 720                 return (rc);
 721
 722         /* Create the common code event queue. */
 723         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
 724             evq->buf_base_id, &evq->common)) != 0)
 725                 goto fail;
 726
 727         SFXGE_EVQ_LOCK(evq);
 728
 729         /* Set the default moderation */
 730         (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
 731
 732         /* Prime the event queue for interrupts */
 733         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 734                 goto fail2;
 735
 736         evq->init_state = SFXGE_EVQ_STARTING;
 737
 738         SFXGE_EVQ_UNLOCK(evq);
 739
 740         /* Wait for the initialization event */
 741         count = 0;
 742         do {
 743                 /* Pause for 100 ms */
 744                 pause("sfxge evq init", hz / 10);
 745
 746                 /* Check to see if the test event has been processed */
 747                 if (evq->init_state == SFXGE_EVQ_STARTED)
 748                         goto done;
 749
 750         } while (++count < 20);
 751
 752         rc = ETIMEDOUT;
 753         goto fail3;
 754
 755 done:
 756         return (0);
 757
 758 fail3:
 759         SFXGE_EVQ_LOCK(evq);
 760         evq->init_state = SFXGE_EVQ_INITIALIZED;
 761 fail2:
 762         SFXGE_EVQ_UNLOCK(evq);
 763         efx_ev_qdestroy(evq->common);
 764 fail:
 765         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 766             EFX_EVQ_NBUFS(evq->entries));
 767
 768         return (rc);
 769 }
 770
 771 void
 772 sfxge_ev_stop(struct sfxge_softc *sc)
 773 {
 774         struct sfxge_intr *intr;
 775         efx_nic_t *enp;
 776         int index;
 777
 778         intr = &sc->intr;
 779         enp = sc->enp;
 780
 781         KASSERT(intr->state == SFXGE_INTR_STARTED,
 782             ("Interrupts not started"));
 783
 784         /* Stop the event queue(s) */
 785         index = sc->evq_count;
 786         while (--index >= 0)
 787                 sfxge_ev_qstop(sc, index);
 788
 789         /* Tear down the event module */
 790         efx_ev_fini(enp);
 791 }
 792
 793 int
 794 sfxge_ev_start(struct sfxge_softc *sc)
 795 {
 796         struct sfxge_intr *intr;
 797         int index;
 798         int rc;
 799
 800         intr = &sc->intr;
 801
 802         KASSERT(intr->state == SFXGE_INTR_STARTED,
 803             ("intr->state != SFXGE_INTR_STARTED"));
 804
 805         /* Initialize the event module */
 806         if ((rc = efx_ev_init(sc->enp)) != 0)
 807                 return (rc);
 808
 809         /* Start the event queues */
 810         for (index = 0; index < sc->evq_count; index++) {
 811                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 812                         goto fail;
 813         }
 814
 815         return (0);
 816
 817 fail:
 818         /* Stop the event queue(s) */
 819         while (--index >= 0)
 820                 sfxge_ev_qstop(sc, index);
 821
 822         /* Tear down the event module */
 823         efx_ev_fini(sc->enp);
 824
 825         return (rc);
 826 }
 827
 828 static void
 829 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 830 {
 831         struct sfxge_evq *evq;
 832
 833         evq = sc->evq[index];
 834
 835         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 836             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 837         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 838
 839         sfxge_dma_free(&evq->mem);
 840
 841         sc->evq[index] = NULL;
 842
 843         SFXGE_EVQ_LOCK_DESTROY(evq);
 844
 845         free(evq, M_SFXGE);
 846 }
 847
 848 static int
 849 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 850 {
 851         struct sfxge_evq *evq;
 852         efsys_mem_t *esmp;
 853         int rc;
 854
 855         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 856
 857         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 858         evq->sc = sc;
 859         evq->index = index;
 860         sc->evq[index] = evq;
 861         esmp = &evq->mem;
 862
 863         /* Build an event queue with room for one event per tx and rx buffer,
 864          * plus some extra for link state events and MCDI completions.
 865          * There are three tx queues in the first event queue and one in
 866          * other.
 867          */
 868         if (index == 0)
 869                 evq->entries =
 870                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 871                                            3 * sc->txq_entries +
 872                                            128);
 873         else
 874                 evq->entries =
 875                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 876                                            sc->txq_entries +
 877                                            128);
 878
 879         /* Initialise TX completion list */
 880         evq->txqs = &evq->txq;
 881
 882         /* Allocate DMA space. */
 883         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
 884                 return (rc);
 885
 886         /* Allocate buffer table entries. */
 887         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
 888                                  &evq->buf_base_id);
 889
 890         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
 891
 892         evq->init_state = SFXGE_EVQ_INITIALIZED;
 893
 894         return (0);
 895 }
 896
 897 void
 898 sfxge_ev_fini(struct sfxge_softc *sc)
 899 {
 900         struct sfxge_intr *intr;
 901         int index;
 902
 903         intr = &sc->intr;
 904
 905         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 906             ("intr->state != SFXGE_INTR_INITIALIZED"));
 907
 908         sc->ev_moderation = 0;
 909
 910         /* Tear down the event queue(s). */
 911         index = sc->evq_count;
 912         while (--index >= 0)
 913                 sfxge_ev_qfini(sc, index);
 914
 915         sc->evq_count = 0;
 916 }
 917
 918 int
 919 sfxge_ev_init(struct sfxge_softc *sc)
 920 {
 921         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 922         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 923         struct sfxge_intr *intr;
 924         int index;
 925         int rc;
 926
 927         intr = &sc->intr;
 928
 929         sc->evq_count = intr->n_alloc;
 930
 931         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 932             ("intr->state != SFXGE_INTR_INITIALIZED"));
 933
 934         /* Set default interrupt moderation; add a sysctl to
 935          * read and change it.
 936          */
 937         sc->ev_moderation = SFXGE_MODERATION;
 938         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 939                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 940                         sc, 0, sfxge_int_mod_handler, "IU",
 941                         "sfxge interrupt moderation (us)");
 942
 943         /*
 944          * Initialize the event queue(s) - one per interrupt.
 945          */
 946         for (index = 0; index < sc->evq_count; index++) {
 947                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 948                         goto fail;
 949         }
 950
 951 #if EFSYS_OPT_QSTATS
 952         sfxge_ev_stat_init(sc);
 953 #endif
 954
 955         return (0);
 956
 957 fail:
 958         while (--index >= 0)
 959                 sfxge_ev_qfini(sc, index);
 960
 961         sc->evq_count = 0;
 962         return (rc);
 963 }