sys/dev/sfxge/sfxge_ev.c

   1 /*-
   2  * Copyright (c) 2010-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright notice,
  12  *    this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  *    this list of conditions and the following disclaimer in the documentation
  15  *    and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  19  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  24  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  25  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  27  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28  *
  29  * The views and conclusions contained in the software and documentation are
  30  * those of the authors and should not be interpreted as representing official
  31  * policies, either expressed or implied, of the FreeBSD Project.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include <sys/param.h>
  38 #include <sys/kernel.h>
  39 #include <sys/malloc.h>
  40 #include <sys/param.h>
  41 #include <sys/queue.h>
  42 #include <sys/systm.h>
  43 #include <sys/taskqueue.h>
  44
  45 #include "common/efx.h"
  46
  47 #include "sfxge.h"
  48
  49 static void
  50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
  51 {
  52         struct sfxge_softc *sc;
  53         unsigned int index;
  54         struct sfxge_rxq *rxq;
  55         struct sfxge_txq *txq;
  56
  57         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
  58
  59         sc = evq->sc;
  60         index = evq->index;
  61         rxq = sc->rxq[index];
  62
  63         if ((txq = evq->txq) != NULL) {
  64                 evq->txq = NULL;
  65                 evq->txqs = &(evq->txq);
  66
  67                 do {
  68                         struct sfxge_txq *next;
  69
  70                         next = txq->next;
  71                         txq->next = NULL;
  72
  73                         KASSERT(txq->evq_index == index,
  74                             ("txq->evq_index != index"));
  75
  76                         if (txq->pending != txq->completed)
  77                                 sfxge_tx_qcomplete(txq, evq);
  78
  79                         txq = next;
  80                 } while (txq != NULL);
  81         }
  82
  83         if (rxq->pending != rxq->completed)
  84                 sfxge_rx_qcomplete(rxq, eop);
  85 }
  86
  87 static boolean_t
  88 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
  89             uint16_t flags)
  90 {
  91         struct sfxge_evq *evq;
  92         struct sfxge_softc *sc;
  93         struct sfxge_rxq *rxq;
  94         unsigned int stop;
  95         unsigned int delta;
  96         struct sfxge_rx_sw_desc *rx_desc;
  97
  98         evq = arg;
  99         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 100
 101         sc = evq->sc;
 102
 103         if (evq->exception)
 104                 goto done;
 105
 106         rxq = sc->rxq[label];
 107         KASSERT(rxq != NULL, ("rxq == NULL"));
 108         KASSERT(evq->index == rxq->index,
 109             ("evq->index != rxq->index"));
 110
 111         if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
 112                 goto done;
 113
 114         stop = (id + 1) & rxq->ptr_mask;
 115         id = rxq->pending & rxq->ptr_mask;
 116         delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
 117         rxq->pending += delta;
 118
 119         if (delta != 1) {
 120                 if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) ||
 121                     (delta <= 0) ||
 122                     (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
 123                         evq->exception = B_TRUE;
 124
 125                         device_printf(sc->dev, "RX completion out of order"
 126                                                   " (id=%#x delta=%u flags=%#x); resetting\n",
 127                                                   id, delta, flags);
 128                         sfxge_schedule_reset(sc);
 129
 130                         goto done;
 131                 }
 132         }
 133
 134         rx_desc = &rxq->queue[id];
 135
 136         prefetch_read_many(rx_desc->mbuf);
 137
 138         for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
 139                 rx_desc = &rxq->queue[id];
 140                 KASSERT(rx_desc->flags == EFX_DISCARD,
 141                                 ("rx_desc->flags != EFX_DISCARD"));
 142                 rx_desc->flags = flags;
 143
 144                 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
 145                 rx_desc->size = (uint16_t)size;
 146         }
 147
 148         evq->rx_done++;
 149
 150         if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
 151                 sfxge_ev_qcomplete(evq, B_FALSE);
 152
 153 done:
 154         return (evq->rx_done >= SFXGE_EV_BATCH);
 155 }
 156
 157 static boolean_t
 158 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
 159 {
 160         struct sfxge_evq *evq;
 161         struct sfxge_softc *sc;
 162
 163         evq = (struct sfxge_evq *)arg;
 164         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 165
 166         sc = evq->sc;
 167
 168         DBGPRINT(sc->dev, "[%d] %s", evq->index,
 169                           (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
 170                           (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
 171                           (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
 172                           (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
 173                           (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
 174                           (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
 175                           (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
 176                           (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
 177                           (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
 178                           "UNKNOWN");
 179
 180         evq->exception = B_TRUE;
 181
 182         if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
 183                 device_printf(sc->dev,
 184                               "hardware exception (code=%u); resetting\n",
 185                               code);
 186                 sfxge_schedule_reset(sc);
 187         }
 188
 189         return (B_FALSE);
 190 }
 191
 192 static boolean_t
 193 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
 194 {
 195         struct sfxge_evq *evq;
 196         struct sfxge_softc *sc;
 197         struct sfxge_rxq *rxq;
 198         unsigned int index;
 199         unsigned int label;
 200         uint16_t magic;
 201
 202         evq = (struct sfxge_evq *)arg;
 203         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 204
 205         sc = evq->sc;
 206         rxq = sc->rxq[rxq_index];
 207
 208         KASSERT(rxq != NULL, ("rxq == NULL"));
 209
 210         /* Resend a software event on the correct queue */
 211         index = rxq->index;
 212         if (index == evq->index) {
 213                 sfxge_rx_qflush_done(rxq);
 214                 return (B_FALSE);
 215         }
 216
 217         evq = sc->evq[index];
 218
 219         label = rxq_index;
 220         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 221             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
 222         magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
 223
 224         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 225             ("evq not started"));
 226         efx_ev_qpost(evq->common, magic);
 227
 228         return (B_FALSE);
 229 }
 230
 231 static boolean_t
 232 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
 233 {
 234         struct sfxge_evq *evq;
 235         struct sfxge_softc *sc;
 236         struct sfxge_rxq *rxq;
 237         unsigned int index;
 238         unsigned int label;
 239         uint16_t magic;
 240
 241         evq = (struct sfxge_evq *)arg;
 242         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 243
 244         sc = evq->sc;
 245         rxq = sc->rxq[rxq_index];
 246
 247         KASSERT(rxq != NULL, ("rxq == NULL"));
 248
 249         /* Resend a software event on the correct queue */
 250         index = rxq->index;
 251         evq = sc->evq[index];
 252
 253         label = rxq_index;
 254         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 255             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 256         magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
 257
 258         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 259             ("evq not started"));
 260         efx_ev_qpost(evq->common, magic);
 261
 262         return (B_FALSE);
 263 }
 264
 265 static struct sfxge_txq *
 266 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
 267 {
 268         unsigned int index;
 269
 270         KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
 271             (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label"));
 272         index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES);
 273         return (evq->sc->txq[index]);
 274 }
 275
 276 static boolean_t
 277 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
 278 {
 279         struct sfxge_evq *evq;
 280         struct sfxge_txq *txq;
 281         unsigned int stop;
 282         unsigned int delta;
 283
 284         evq = (struct sfxge_evq *)arg;
 285         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 286
 287         txq = sfxge_get_txq_by_label(evq, label);
 288
 289         KASSERT(txq != NULL, ("txq == NULL"));
 290         KASSERT(evq->index == txq->evq_index,
 291             ("evq->index != txq->evq_index"));
 292
 293         if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
 294                 goto done;
 295
 296         stop = (id + 1) & txq->ptr_mask;
 297         id = txq->pending & txq->ptr_mask;
 298
 299         delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
 300         txq->pending += delta;
 301
 302         evq->tx_done++;
 303
 304         if (txq->next == NULL &&
 305             evq->txqs != &(txq->next)) {
 306                 *(evq->txqs) = txq;
 307                 evq->txqs = &(txq->next);
 308         }
 309
 310         if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
 311                 sfxge_tx_qcomplete(txq, evq);
 312
 313 done:
 314         return (evq->tx_done >= SFXGE_EV_BATCH);
 315 }
 316
 317 static boolean_t
 318 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
 319 {
 320         struct sfxge_evq *evq;
 321         struct sfxge_softc *sc;
 322         struct sfxge_txq *txq;
 323         unsigned int label;
 324         uint16_t magic;
 325
 326         evq = (struct sfxge_evq *)arg;
 327         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 328
 329         sc = evq->sc;
 330         txq = sc->txq[txq_index];
 331
 332         KASSERT(txq != NULL, ("txq == NULL"));
 333         KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
 334             ("txq not initialized"));
 335
 336         if (txq->evq_index == evq->index) {
 337                 sfxge_tx_qflush_done(txq);
 338                 return (B_FALSE);
 339         }
 340
 341         /* Resend a software event on the correct queue */
 342         evq = sc->evq[txq->evq_index];
 343
 344         label = txq->type;
 345         KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
 346             ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
 347         magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
 348
 349         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 350             ("evq not started"));
 351         efx_ev_qpost(evq->common, magic);
 352
 353         return (B_FALSE);
 354 }
 355
 356 static boolean_t
 357 sfxge_ev_software(void *arg, uint16_t magic)
 358 {
 359         struct sfxge_evq *evq;
 360         struct sfxge_softc *sc;
 361         unsigned int label;
 362
 363         evq = (struct sfxge_evq *)arg;
 364         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 365
 366         sc = evq->sc;
 367
 368         label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
 369         magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
 370
 371         switch (magic) {
 372         case SFXGE_MAGIC_RX_QFLUSH_DONE: {
 373                 struct sfxge_rxq *rxq = sc->rxq[label];
 374
 375                 KASSERT(rxq != NULL, ("rxq == NULL"));
 376                 KASSERT(evq->index == rxq->index,
 377                     ("evq->index != rxq->index"));
 378
 379                 sfxge_rx_qflush_done(rxq);
 380                 break;
 381         }
 382         case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
 383                 struct sfxge_rxq *rxq = sc->rxq[label];
 384
 385                 KASSERT(rxq != NULL, ("rxq == NULL"));
 386                 KASSERT(evq->index == rxq->index,
 387                     ("evq->index != rxq->index"));
 388
 389                 sfxge_rx_qflush_failed(rxq);
 390                 break;
 391         }
 392         case SFXGE_MAGIC_RX_QREFILL: {
 393                 struct sfxge_rxq *rxq = sc->rxq[label];
 394
 395                 KASSERT(rxq != NULL, ("rxq == NULL"));
 396                 KASSERT(evq->index == rxq->index,
 397                     ("evq->index != rxq->index"));
 398
 399                 sfxge_rx_qrefill(rxq);
 400                 break;
 401         }
 402         case SFXGE_MAGIC_TX_QFLUSH_DONE: {
 403                 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
 404
 405                 KASSERT(txq != NULL, ("txq == NULL"));
 406                 KASSERT(evq->index == txq->evq_index,
 407                     ("evq->index != txq->evq_index"));
 408
 409                 sfxge_tx_qflush_done(txq);
 410                 break;
 411         }
 412         default:
 413                 break;
 414         }
 415
 416         return (B_FALSE);
 417 }
 418
 419 static boolean_t
 420 sfxge_ev_sram(void *arg, uint32_t code)
 421 {
 422         (void)arg;
 423         (void)code;
 424
 425         switch (code) {
 426         case EFX_SRAM_UPDATE:
 427                 EFSYS_PROBE(sram_update);
 428                 break;
 429
 430         case EFX_SRAM_CLEAR:
 431                 EFSYS_PROBE(sram_clear);
 432                 break;
 433
 434         case EFX_SRAM_ILLEGAL_CLEAR:
 435                 EFSYS_PROBE(sram_illegal_clear);
 436                 break;
 437
 438         default:
 439                 KASSERT(B_FALSE, ("Impossible SRAM event"));
 440                 break;
 441         }
 442
 443         return (B_FALSE);
 444 }
 445
 446 static boolean_t
 447 sfxge_ev_timer(void *arg, uint32_t index)
 448 {
 449         (void)arg;
 450         (void)index;
 451
 452         return (B_FALSE);
 453 }
 454
 455 static boolean_t
 456 sfxge_ev_wake_up(void *arg, uint32_t index)
 457 {
 458         (void)arg;
 459         (void)index;
 460
 461         return (B_FALSE);
 462 }
 463
 464 #if EFSYS_OPT_QSTATS
 465
 466 static void
 467 sfxge_ev_stat_update(struct sfxge_softc *sc)
 468 {
 469         struct sfxge_evq *evq;
 470         unsigned int index;
 471         clock_t now;
 472
 473         SFXGE_ADAPTER_LOCK(sc);
 474
 475         if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED))
 476                 goto out;
 477
 478         now = ticks;
 479         if (now - sc->ev_stats_update_time < hz)
 480                 goto out;
 481
 482         sc->ev_stats_update_time = now;
 483
 484         /* Add event counts from each event queue in turn */
 485         for (index = 0; index < sc->evq_count; index++) {
 486                 evq = sc->evq[index];
 487                 SFXGE_EVQ_LOCK(evq);
 488                 efx_ev_qstats_update(evq->common, sc->ev_stats);
 489                 SFXGE_EVQ_UNLOCK(evq);
 490         }
 491 out:
 492         SFXGE_ADAPTER_UNLOCK(sc);
 493 }
 494
 495 static int
 496 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
 497 {
 498         struct sfxge_softc *sc = arg1;
 499         unsigned int id = arg2;
 500
 501         sfxge_ev_stat_update(sc);
 502
 503         return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
 504 }
 505
 506 static void
 507 sfxge_ev_stat_init(struct sfxge_softc *sc)
 508 {
 509         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
 510         struct sysctl_oid_list *stat_list;
 511         unsigned int id;
 512         char name[40];
 513
 514         stat_list = SYSCTL_CHILDREN(sc->stats_node);
 515
 516         for (id = 0; id < EV_NQSTATS; id++) {
 517                 snprintf(name, sizeof(name), "ev_%s",
 518                          efx_ev_qstat_name(sc->enp, id));
 519                 SYSCTL_ADD_PROC(
 520                         ctx, stat_list,
 521                         OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
 522                         sc, id, sfxge_ev_stat_handler, "Q",
 523                         "");
 524         }
 525 }
 526
 527 #endif /* EFSYS_OPT_QSTATS */
 528
 529 static void
 530 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
 531 {
 532         struct sfxge_evq *evq;
 533         efx_evq_t *eep;
 534
 535         evq = sc->evq[idx];
 536         eep = evq->common;
 537
 538         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 539             ("evq->init_state != SFXGE_EVQ_STARTED"));
 540
 541         (void)efx_ev_qmoderate(eep, us);
 542 }
 543
 544 static int
 545 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
 546 {
 547         struct sfxge_softc *sc = arg1;
 548         struct sfxge_intr *intr = &sc->intr;
 549         unsigned int moderation;
 550         int error;
 551         unsigned int index;
 552
 553         SFXGE_ADAPTER_LOCK(sc);
 554
 555         if (req->newptr != NULL) {
 556                 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
 557                     != 0)
 558                         goto out;
 559
 560                 /* We may not be calling efx_ev_qmoderate() now,
 561                  * so we have to range-check the value ourselves.
 562                  */
 563                 if (moderation >
 564                     efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
 565                         error = EINVAL;
 566                         goto out;
 567                 }
 568
 569                 sc->ev_moderation = moderation;
 570                 if (intr->state == SFXGE_INTR_STARTED) {
 571                         for (index = 0; index < sc->evq_count; index++)
 572                                 sfxge_ev_qmoderate(sc, index, moderation);
 573                 }
 574         } else {
 575                 error = SYSCTL_OUT(req, &sc->ev_moderation,
 576                                    sizeof(sc->ev_moderation));
 577         }
 578
 579 out:
 580         SFXGE_ADAPTER_UNLOCK(sc);
 581
 582         return (error);
 583 }
 584
 585 static boolean_t
 586 sfxge_ev_initialized(void *arg)
 587 {
 588         struct sfxge_evq *evq;
 589
 590         evq = (struct sfxge_evq *)arg;
 591         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 592
 593         /* Init done events may be duplicated on 7xxx */
 594         KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
 595                 evq->init_state == SFXGE_EVQ_STARTED,
 596             ("evq not starting"));
 597
 598         evq->init_state = SFXGE_EVQ_STARTED;
 599
 600         return (0);
 601 }
 602
 603 static boolean_t
 604 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
 605 {
 606         struct sfxge_evq *evq;
 607         struct sfxge_softc *sc;
 608
 609         evq = (struct sfxge_evq *)arg;
 610         SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
 611
 612         sc = evq->sc;
 613
 614         sfxge_mac_link_update(sc, link_mode);
 615
 616         return (0);
 617 }
 618
 619 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
 620         .eec_initialized        = sfxge_ev_initialized,
 621         .eec_rx                 = sfxge_ev_rx,
 622         .eec_tx                 = sfxge_ev_tx,
 623         .eec_exception          = sfxge_ev_exception,
 624         .eec_rxq_flush_done     = sfxge_ev_rxq_flush_done,
 625         .eec_rxq_flush_failed   = sfxge_ev_rxq_flush_failed,
 626         .eec_txq_flush_done     = sfxge_ev_txq_flush_done,
 627         .eec_software           = sfxge_ev_software,
 628         .eec_sram               = sfxge_ev_sram,
 629         .eec_wake_up            = sfxge_ev_wake_up,
 630         .eec_timer              = sfxge_ev_timer,
 631         .eec_link_change        = sfxge_ev_link_change,
 632 };
 633
 634
 635 int
 636 sfxge_ev_qpoll(struct sfxge_evq *evq)
 637 {
 638         int rc;
 639
 640         SFXGE_EVQ_LOCK(evq);
 641
 642         if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
 643                             evq->init_state != SFXGE_EVQ_STARTED)) {
 644                 rc = EINVAL;
 645                 goto fail;
 646         }
 647
 648         /* Synchronize the DMA memory for reading */
 649         bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
 650             BUS_DMASYNC_POSTREAD);
 651
 652         KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
 653         KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
 654         KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
 655         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 656
 657         /* Poll the queue */
 658         efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
 659
 660         evq->rx_done = 0;
 661         evq->tx_done = 0;
 662
 663         /* Perform any pending completion processing */
 664         sfxge_ev_qcomplete(evq, B_TRUE);
 665
 666         /* Re-prime the event queue for interrupts */
 667         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 668                 goto fail;
 669
 670         SFXGE_EVQ_UNLOCK(evq);
 671
 672         return (0);
 673
 674 fail:
 675         SFXGE_EVQ_UNLOCK(evq);
 676         return (rc);
 677 }
 678
 679 static void
 680 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
 681 {
 682         struct sfxge_evq *evq;
 683
 684         evq = sc->evq[index];
 685
 686         KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
 687             ("evq->init_state != SFXGE_EVQ_STARTED"));
 688
 689         SFXGE_EVQ_LOCK(evq);
 690         evq->init_state = SFXGE_EVQ_INITIALIZED;
 691         evq->read_ptr = 0;
 692         evq->exception = B_FALSE;
 693
 694 #if EFSYS_OPT_QSTATS
 695         /* Add event counts before discarding the common evq state */
 696         efx_ev_qstats_update(evq->common, sc->ev_stats);
 697 #endif
 698
 699         efx_ev_qdestroy(evq->common);
 700         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 701             EFX_EVQ_NBUFS(evq->entries));
 702         SFXGE_EVQ_UNLOCK(evq);
 703 }
 704
 705 static int
 706 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
 707 {
 708         struct sfxge_evq *evq;
 709         efsys_mem_t *esmp;
 710         int count;
 711         int rc;
 712
 713         evq = sc->evq[index];
 714         esmp = &evq->mem;
 715
 716         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 717             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 718
 719         /* Clear all events. */
 720         (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
 721
 722         /* Program the buffer table. */
 723         if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
 724             EFX_EVQ_NBUFS(evq->entries))) != 0)
 725                 return (rc);
 726
 727         /* Create the common code event queue. */
 728         if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
 729             evq->buf_base_id, &evq->common)) != 0)
 730                 goto fail;
 731
 732         SFXGE_EVQ_LOCK(evq);
 733
 734         /* Set the default moderation */
 735         (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
 736
 737         /* Prime the event queue for interrupts */
 738         if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
 739                 goto fail2;
 740
 741         evq->init_state = SFXGE_EVQ_STARTING;
 742
 743         SFXGE_EVQ_UNLOCK(evq);
 744
 745         /* Wait for the initialization event */
 746         count = 0;
 747         do {
 748                 /* Pause for 100 ms */
 749                 pause("sfxge evq init", hz / 10);
 750
 751                 /* Check to see if the test event has been processed */
 752                 if (evq->init_state == SFXGE_EVQ_STARTED)
 753                         goto done;
 754
 755         } while (++count < 20);
 756
 757         rc = ETIMEDOUT;
 758         goto fail3;
 759
 760 done:
 761         return (0);
 762
 763 fail3:
 764         SFXGE_EVQ_LOCK(evq);
 765         evq->init_state = SFXGE_EVQ_INITIALIZED;
 766 fail2:
 767         SFXGE_EVQ_UNLOCK(evq);
 768         efx_ev_qdestroy(evq->common);
 769 fail:
 770         efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
 771             EFX_EVQ_NBUFS(evq->entries));
 772
 773         return (rc);
 774 }
 775
 776 void
 777 sfxge_ev_stop(struct sfxge_softc *sc)
 778 {
 779         struct sfxge_intr *intr;
 780         efx_nic_t *enp;
 781         int index;
 782
 783         intr = &sc->intr;
 784         enp = sc->enp;
 785
 786         KASSERT(intr->state == SFXGE_INTR_STARTED,
 787             ("Interrupts not started"));
 788
 789         /* Stop the event queue(s) */
 790         index = sc->evq_count;
 791         while (--index >= 0)
 792                 sfxge_ev_qstop(sc, index);
 793
 794         /* Tear down the event module */
 795         efx_ev_fini(enp);
 796 }
 797
 798 int
 799 sfxge_ev_start(struct sfxge_softc *sc)
 800 {
 801         struct sfxge_intr *intr;
 802         int index;
 803         int rc;
 804
 805         intr = &sc->intr;
 806
 807         KASSERT(intr->state == SFXGE_INTR_STARTED,
 808             ("intr->state != SFXGE_INTR_STARTED"));
 809
 810         /* Initialize the event module */
 811         if ((rc = efx_ev_init(sc->enp)) != 0)
 812                 return (rc);
 813
 814         /* Start the event queues */
 815         for (index = 0; index < sc->evq_count; index++) {
 816                 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
 817                         goto fail;
 818         }
 819
 820         return (0);
 821
 822 fail:
 823         /* Stop the event queue(s) */
 824         while (--index >= 0)
 825                 sfxge_ev_qstop(sc, index);
 826
 827         /* Tear down the event module */
 828         efx_ev_fini(sc->enp);
 829
 830         return (rc);
 831 }
 832
 833 static void
 834 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
 835 {
 836         struct sfxge_evq *evq;
 837
 838         evq = sc->evq[index];
 839
 840         KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
 841             ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
 842         KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
 843
 844         sfxge_dma_free(&evq->mem);
 845
 846         sc->evq[index] = NULL;
 847
 848         SFXGE_EVQ_LOCK_DESTROY(evq);
 849
 850         free(evq, M_SFXGE);
 851 }
 852
 853 static int
 854 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
 855 {
 856         struct sfxge_evq *evq;
 857         efsys_mem_t *esmp;
 858         int rc;
 859
 860         KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
 861
 862         evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
 863         evq->sc = sc;
 864         evq->index = index;
 865         sc->evq[index] = evq;
 866         esmp = &evq->mem;
 867
 868         /* Build an event queue with room for one event per tx and rx buffer,
 869          * plus some extra for link state events and MCDI completions.
 870          * There are three tx queues in the first event queue and one in
 871          * other.
 872          */
 873         if (index == 0)
 874                 evq->entries =
 875                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 876                                            3 * sc->txq_entries +
 877                                            128);
 878         else
 879                 evq->entries =
 880                         ROUNDUP_POW_OF_TWO(sc->rxq_entries +
 881                                            sc->txq_entries +
 882                                            128);
 883
 884         /* Initialise TX completion list */
 885         evq->txqs = &evq->txq;
 886
 887         /* Allocate DMA space. */
 888         if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
 889                 return (rc);
 890
 891         /* Allocate buffer table entries. */
 892         sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
 893                                  &evq->buf_base_id);
 894
 895         SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
 896
 897         evq->init_state = SFXGE_EVQ_INITIALIZED;
 898
 899         return (0);
 900 }
 901
 902 void
 903 sfxge_ev_fini(struct sfxge_softc *sc)
 904 {
 905         struct sfxge_intr *intr;
 906         int index;
 907
 908         intr = &sc->intr;
 909
 910         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 911             ("intr->state != SFXGE_INTR_INITIALIZED"));
 912
 913         sc->ev_moderation = 0;
 914
 915         /* Tear down the event queue(s). */
 916         index = sc->evq_count;
 917         while (--index >= 0)
 918                 sfxge_ev_qfini(sc, index);
 919
 920         sc->evq_count = 0;
 921 }
 922
 923 int
 924 sfxge_ev_init(struct sfxge_softc *sc)
 925 {
 926         struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
 927         struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
 928         struct sfxge_intr *intr;
 929         int index;
 930         int rc;
 931
 932         intr = &sc->intr;
 933
 934         sc->evq_count = intr->n_alloc;
 935
 936         KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
 937             ("intr->state != SFXGE_INTR_INITIALIZED"));
 938
 939         /* Set default interrupt moderation; add a sysctl to
 940          * read and change it.
 941          */
 942         sc->ev_moderation = SFXGE_MODERATION;
 943         SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
 944                         OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
 945                         sc, 0, sfxge_int_mod_handler, "IU",
 946                         "sfxge interrupt moderation (us)");
 947
 948         /*
 949          * Initialize the event queue(s) - one per interrupt.
 950          */
 951         for (index = 0; index < sc->evq_count; index++) {
 952                 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
 953                         goto fail;
 954         }
 955
 956 #if EFSYS_OPT_QSTATS
 957         sfxge_ev_stat_init(sc);
 958 #endif
 959
 960         return (0);
 961
 962 fail:
 963         while (--index >= 0)
 964                 sfxge_ev_qfini(sc, index);
 965
 966         sc->evq_count = 0;
 967         return (rc);
 968 }