2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2010-2016 Solarflare Communications Inc.
7 * This software was developed in part by Philip Paeps under contract for
8 * Solarflare Communications, Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 * The views and conclusions contained in the software and documentation are
32 * those of the authors and should not be interpreted as representing official
33 * policies, either expressed or implied, of the FreeBSD Project.
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
39 #include <sys/param.h>
40 #include <sys/kernel.h>
41 #include <sys/malloc.h>
42 #include <sys/param.h>
43 #include <sys/queue.h>
44 #include <sys/systm.h>
45 #include <sys/taskqueue.h>
47 #include "common/efx.h"
52 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
54 struct sfxge_softc *sc;
56 struct sfxge_rxq *rxq;
57 struct sfxge_txq *txq;
59 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
65 if ((txq = evq->txq) != NULL) {
67 evq->txqs = &(evq->txq);
70 struct sfxge_txq *next;
75 KASSERT(txq->evq_index == index,
76 ("txq->evq_index != index"));
78 if (txq->pending != txq->completed)
79 sfxge_tx_qcomplete(txq, evq);
82 } while (txq != NULL);
85 if (rxq->pending != rxq->completed)
86 sfxge_rx_qcomplete(rxq, eop);
89 static struct sfxge_rxq *
90 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
92 struct sfxge_rxq *rxq;
94 KASSERT(label == 0, ("unexpected rxq label != 0"));
96 rxq = evq->sc->rxq[evq->index];
98 KASSERT(rxq != NULL, ("rxq == NULL"));
99 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
105 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
108 struct sfxge_evq *evq;
109 struct sfxge_softc *sc;
110 struct sfxge_rxq *rxq;
113 struct sfxge_rx_sw_desc *rx_desc;
116 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
123 rxq = sfxge_get_rxq_by_label(evq, label);
124 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
127 stop = (id + 1) & rxq->ptr_mask;
128 id = rxq->pending & rxq->ptr_mask;
129 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
130 rxq->pending += delta;
134 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
135 evq->exception = B_TRUE;
137 device_printf(sc->dev, "RX completion out of order"
138 " (id=%#x delta=%u flags=%#x); resetting\n",
140 sfxge_schedule_reset(sc);
146 rx_desc = &rxq->queue[id];
148 prefetch_read_many(rx_desc->mbuf);
150 for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
151 rx_desc = &rxq->queue[id];
152 KASSERT(rx_desc->flags == EFX_DISCARD,
153 ("rx_desc->flags != EFX_DISCARD"));
154 rx_desc->flags = flags;
156 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
157 rx_desc->size = (uint16_t)size;
162 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
163 sfxge_ev_qcomplete(evq, B_FALSE);
166 return (evq->rx_done >= SFXGE_EV_BATCH);
170 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
172 struct sfxge_evq *evq;
173 struct sfxge_softc *sc;
175 evq = (struct sfxge_evq *)arg;
176 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
180 DBGPRINT(sc->dev, "[%d] %s", evq->index,
181 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
182 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
183 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
184 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
185 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
186 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
187 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
188 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
189 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
192 evq->exception = B_TRUE;
194 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
195 device_printf(sc->dev,
196 "hardware exception (code=%u); resetting\n",
198 sfxge_schedule_reset(sc);
205 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
207 struct sfxge_evq *evq;
208 struct sfxge_softc *sc;
209 struct sfxge_rxq *rxq;
213 evq = (struct sfxge_evq *)arg;
214 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
217 rxq = sc->rxq[rxq_index];
219 KASSERT(rxq != NULL, ("rxq == NULL"));
221 /* Resend a software event on the correct queue */
223 if (index == evq->index) {
224 sfxge_rx_qflush_done(rxq);
228 evq = sc->evq[index];
229 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
231 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
232 ("evq not started"));
233 efx_ev_qpost(evq->common, magic);
239 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
241 struct sfxge_evq *evq;
242 struct sfxge_softc *sc;
243 struct sfxge_rxq *rxq;
247 evq = (struct sfxge_evq *)arg;
248 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
251 rxq = sc->rxq[rxq_index];
253 KASSERT(rxq != NULL, ("rxq == NULL"));
255 /* Resend a software event on the correct queue */
257 evq = sc->evq[index];
258 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
260 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
261 ("evq not started"));
262 efx_ev_qpost(evq->common, magic);
267 static struct sfxge_txq *
268 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
272 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) :
273 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
274 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)),
275 ("unexpected txq label"));
277 index = (evq->index == 0) ?
278 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc));
279 return (evq->sc->txq[index]);
283 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
285 struct sfxge_evq *evq;
286 struct sfxge_txq *txq;
290 evq = (struct sfxge_evq *)arg;
291 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
293 txq = sfxge_get_txq_by_label(evq, label);
295 KASSERT(txq != NULL, ("txq == NULL"));
296 KASSERT(evq->index == txq->evq_index,
297 ("evq->index != txq->evq_index"));
299 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
302 stop = (id + 1) & txq->ptr_mask;
303 id = txq->pending & txq->ptr_mask;
305 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
306 txq->pending += delta;
310 if (txq->next == NULL &&
311 evq->txqs != &(txq->next)) {
313 evq->txqs = &(txq->next);
316 if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
317 sfxge_tx_qcomplete(txq, evq);
320 return (evq->tx_done >= SFXGE_EV_BATCH);
324 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
326 struct sfxge_evq *evq;
327 struct sfxge_softc *sc;
328 struct sfxge_txq *txq;
331 evq = (struct sfxge_evq *)arg;
332 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
335 txq = sc->txq[txq_index];
337 KASSERT(txq != NULL, ("txq == NULL"));
338 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
339 ("txq not initialized"));
341 if (txq->evq_index == evq->index) {
342 sfxge_tx_qflush_done(txq);
346 /* Resend a software event on the correct queue */
347 evq = sc->evq[txq->evq_index];
348 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
350 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
351 ("evq not started"));
352 efx_ev_qpost(evq->common, magic);
358 sfxge_ev_software(void *arg, uint16_t magic)
360 struct sfxge_evq *evq;
361 struct sfxge_softc *sc;
364 evq = (struct sfxge_evq *)arg;
365 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
369 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
370 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
373 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
374 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
377 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
378 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
381 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
382 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
385 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
386 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
388 KASSERT(txq != NULL, ("txq == NULL"));
389 KASSERT(evq->index == txq->evq_index,
390 ("evq->index != txq->evq_index"));
392 sfxge_tx_qflush_done(txq);
403 sfxge_ev_sram(void *arg, uint32_t code)
409 case EFX_SRAM_UPDATE:
410 EFSYS_PROBE(sram_update);
414 EFSYS_PROBE(sram_clear);
417 case EFX_SRAM_ILLEGAL_CLEAR:
418 EFSYS_PROBE(sram_illegal_clear);
422 KASSERT(B_FALSE, ("Impossible SRAM event"));
430 sfxge_ev_timer(void *arg, uint32_t index)
439 sfxge_ev_wake_up(void *arg, uint32_t index)
450 sfxge_evq_stat_update(struct sfxge_evq *evq)
456 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED))
460 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz)
463 evq->stats_update_time = now;
464 efx_ev_qstats_update(evq->common, evq->stats);
467 SFXGE_EVQ_UNLOCK(evq);
471 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS)
473 struct sfxge_evq *evq = arg1;
474 struct sfxge_softc *sc = evq->sc;
475 unsigned int id = arg2;
477 SFXGE_ADAPTER_LOCK(sc);
479 sfxge_evq_stat_update(evq);
481 SFXGE_ADAPTER_UNLOCK(sc);
483 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id])));
487 sfxge_evq_stat_init(struct sfxge_evq *evq)
489 struct sfxge_softc *sc = evq->sc;
490 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
492 struct sysctl_oid *evq_stats_node;
495 snprintf(name, sizeof(name), "%u", evq->index);
496 evq_stats_node = SYSCTL_ADD_NODE(ctx,
497 SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name,
498 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
499 if (evq_stats_node == NULL)
502 for (id = 0; id < EV_NQSTATS; id++) {
503 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node),
504 OID_AUTO, efx_ev_qstat_name(sc->enp, id),
505 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id,
506 sfxge_evq_stat_handler, "Q", "");
513 sfxge_ev_stat_update(struct sfxge_softc *sc)
515 struct sfxge_evq *evq;
520 SFXGE_ADAPTER_LOCK(sc);
523 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz)
526 sc->ev_stats_update_time = now;
528 memset(sc->ev_stats, 0, sizeof(sc->ev_stats));
530 /* Update and add event counts from each event queue in turn */
531 for (index = 0; index < sc->evq_count; index++) {
532 evq = sc->evq[index];
533 sfxge_evq_stat_update(evq);
534 for (id = 0; id < EV_NQSTATS; id++)
535 sc->ev_stats[id] += evq->stats[id];
538 SFXGE_ADAPTER_UNLOCK(sc);
542 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
544 struct sfxge_softc *sc = arg1;
545 unsigned int id = arg2;
547 sfxge_ev_stat_update(sc);
549 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
553 sfxge_ev_stat_init(struct sfxge_softc *sc)
555 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
556 struct sysctl_oid_list *stat_list;
560 stat_list = SYSCTL_CHILDREN(sc->stats_node);
562 for (id = 0; id < EV_NQSTATS; id++) {
563 snprintf(name, sizeof(name), "ev_%s",
564 efx_ev_qstat_name(sc->enp, id));
565 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name,
566 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
567 sc, id, sfxge_ev_stat_handler, "Q", "");
571 #endif /* EFSYS_OPT_QSTATS */
574 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
576 struct sfxge_evq *evq;
582 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
583 ("evq->init_state != SFXGE_EVQ_STARTED"));
585 (void)efx_ev_qmoderate(eep, us);
589 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
591 struct sfxge_softc *sc = arg1;
592 struct sfxge_intr *intr = &sc->intr;
593 unsigned int moderation;
597 SFXGE_ADAPTER_LOCK(sc);
599 if (req->newptr != NULL) {
600 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
604 /* We may not be calling efx_ev_qmoderate() now,
605 * so we have to range-check the value ourselves.
608 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
613 sc->ev_moderation = moderation;
614 if (intr->state == SFXGE_INTR_STARTED) {
615 for (index = 0; index < sc->evq_count; index++)
616 sfxge_ev_qmoderate(sc, index, moderation);
619 error = SYSCTL_OUT(req, &sc->ev_moderation,
620 sizeof(sc->ev_moderation));
624 SFXGE_ADAPTER_UNLOCK(sc);
630 sfxge_ev_initialized(void *arg)
632 struct sfxge_evq *evq;
634 evq = (struct sfxge_evq *)arg;
635 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
637 /* Init done events may be duplicated on 7xxx */
638 KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
639 evq->init_state == SFXGE_EVQ_STARTED,
640 ("evq not starting"));
642 evq->init_state = SFXGE_EVQ_STARTED;
648 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
650 struct sfxge_evq *evq;
651 struct sfxge_softc *sc;
653 evq = (struct sfxge_evq *)arg;
654 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
658 sfxge_mac_link_update(sc, link_mode);
663 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
664 .eec_initialized = sfxge_ev_initialized,
665 .eec_rx = sfxge_ev_rx,
666 .eec_tx = sfxge_ev_tx,
667 .eec_exception = sfxge_ev_exception,
668 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done,
669 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed,
670 .eec_txq_flush_done = sfxge_ev_txq_flush_done,
671 .eec_software = sfxge_ev_software,
672 .eec_sram = sfxge_ev_sram,
673 .eec_wake_up = sfxge_ev_wake_up,
674 .eec_timer = sfxge_ev_timer,
675 .eec_link_change = sfxge_ev_link_change,
679 sfxge_ev_qpoll(struct sfxge_evq *evq)
685 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
686 evq->init_state != SFXGE_EVQ_STARTED)) {
691 /* Synchronize the DMA memory for reading */
692 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
693 BUS_DMASYNC_POSTREAD);
695 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
696 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
697 KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
698 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
701 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
706 /* Perform any pending completion processing */
707 sfxge_ev_qcomplete(evq, B_TRUE);
709 /* Re-prime the event queue for interrupts */
710 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
713 SFXGE_EVQ_UNLOCK(evq);
718 SFXGE_EVQ_UNLOCK(evq);
723 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
725 struct sfxge_evq *evq;
727 evq = sc->evq[index];
729 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
730 ("evq->init_state != SFXGE_EVQ_STARTED"));
733 evq->init_state = SFXGE_EVQ_INITIALIZED;
735 evq->exception = B_FALSE;
738 /* Add event counts before discarding the common evq state */
739 efx_ev_qstats_update(evq->common, evq->stats);
742 efx_ev_qdestroy(evq->common);
743 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
744 EFX_EVQ_NBUFS(evq->entries));
745 SFXGE_EVQ_UNLOCK(evq);
749 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
751 struct sfxge_evq *evq;
756 evq = sc->evq[index];
759 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
760 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
762 /* Clear all events. */
763 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
765 /* Program the buffer table. */
766 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
767 EFX_EVQ_NBUFS(evq->entries))) != 0)
770 /* Create the common code event queue. */
771 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
772 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
778 /* Prime the event queue for interrupts */
779 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
782 evq->init_state = SFXGE_EVQ_STARTING;
784 SFXGE_EVQ_UNLOCK(evq);
786 /* Wait for the initialization event */
789 /* Pause for 100 ms */
790 pause("sfxge evq init", hz / 10);
792 /* Check to see if the test event has been processed */
793 if (evq->init_state == SFXGE_EVQ_STARTED)
796 } while (++count < 20);
806 evq->init_state = SFXGE_EVQ_INITIALIZED;
808 SFXGE_EVQ_UNLOCK(evq);
809 efx_ev_qdestroy(evq->common);
811 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
812 EFX_EVQ_NBUFS(evq->entries));
818 sfxge_ev_stop(struct sfxge_softc *sc)
820 struct sfxge_intr *intr;
827 KASSERT(intr->state == SFXGE_INTR_STARTED,
828 ("Interrupts not started"));
830 /* Stop the event queue(s) */
831 index = sc->evq_count;
833 sfxge_ev_qstop(sc, index);
835 /* Tear down the event module */
840 sfxge_ev_start(struct sfxge_softc *sc)
842 struct sfxge_intr *intr;
848 KASSERT(intr->state == SFXGE_INTR_STARTED,
849 ("intr->state != SFXGE_INTR_STARTED"));
851 /* Initialize the event module */
852 if ((rc = efx_ev_init(sc->enp)) != 0)
855 /* Start the event queues */
856 for (index = 0; index < sc->evq_count; index++) {
857 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
864 /* Stop the event queue(s) */
866 sfxge_ev_qstop(sc, index);
868 /* Tear down the event module */
869 efx_ev_fini(sc->enp);
875 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
877 struct sfxge_evq *evq;
879 evq = sc->evq[index];
881 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
882 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
883 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
885 sfxge_dma_free(&evq->mem);
887 sc->evq[index] = NULL;
889 SFXGE_EVQ_LOCK_DESTROY(evq);
895 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
897 struct sfxge_evq *evq;
901 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
903 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
906 sc->evq[index] = evq;
909 /* Build an event queue with room for one event per tx and rx buffer,
910 * plus some extra for link state events and MCDI completions.
911 * There are three tx queues in the first event queue and one in
916 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
917 3 * sc->txq_entries +
921 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
925 /* Initialise TX completion list */
926 evq->txqs = &evq->txq;
928 /* Allocate DMA space. */
929 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
932 /* Allocate buffer table entries. */
933 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
936 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
938 evq->init_state = SFXGE_EVQ_INITIALIZED;
941 rc = sfxge_evq_stat_init(evq);
943 goto fail_evq_stat_init;
950 evq->init_state = SFXGE_EVQ_UNINITIALIZED;
951 SFXGE_EVQ_LOCK_DESTROY(evq);
952 sfxge_dma_free(esmp);
953 sc->evq[index] = NULL;
961 sfxge_ev_fini(struct sfxge_softc *sc)
963 struct sfxge_intr *intr;
968 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
969 ("intr->state != SFXGE_INTR_INITIALIZED"));
971 sc->ev_moderation = 0;
973 /* Tear down the event queue(s). */
974 index = sc->evq_count;
976 sfxge_ev_qfini(sc, index);
982 sfxge_ev_init(struct sfxge_softc *sc)
984 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
985 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
986 struct sfxge_intr *intr;
992 sc->evq_count = intr->n_alloc;
994 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
995 ("intr->state != SFXGE_INTR_INITIALIZED"));
997 /* Set default interrupt moderation; add a sysctl to
998 * read and change it.
1000 sc->ev_moderation = SFXGE_MODERATION;
1001 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
1002 OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1003 sc, 0, sfxge_int_mod_handler, "IU",
1004 "sfxge interrupt moderation (us)");
1006 #if EFSYS_OPT_QSTATS
1007 sc->evqs_stats_node = SYSCTL_ADD_NODE(
1008 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node),
1009 OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1010 "Event queues stats");
1011 if (sc->evqs_stats_node == NULL) {
1013 goto fail_evqs_stats_node;
1018 * Initialize the event queue(s) - one per interrupt.
1020 for (index = 0; index < sc->evq_count; index++) {
1021 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
1025 #if EFSYS_OPT_QSTATS
1026 sfxge_ev_stat_init(sc);
1032 while (--index >= 0)
1033 sfxge_ev_qfini(sc, index);
1035 #if EFSYS_OPT_QSTATS
1036 fail_evqs_stats_node: