2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
36 #include "common/efx.h"
41 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
43 struct sfxge_softc *sc;
45 struct sfxge_rxq *rxq;
46 struct sfxge_txq *txq;
52 if ((txq = evq->txq) != NULL) {
54 evq->txqs = &(evq->txq);
57 struct sfxge_txq *next;
62 KASSERT(txq->evq_index == index,
63 ("txq->evq_index != index"));
65 if (txq->pending != txq->completed)
66 sfxge_tx_qcomplete(txq);
69 } while (txq != NULL);
72 if (rxq->pending != rxq->completed)
73 sfxge_rx_qcomplete(rxq, eop);
77 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
80 struct sfxge_evq *evq;
81 struct sfxge_softc *sc;
82 struct sfxge_rxq *rxq;
83 unsigned int expected;
84 struct sfxge_rx_sw_desc *rx_desc;
93 KASSERT(rxq != NULL, ("rxq == NULL"));
94 KASSERT(evq->index == rxq->index,
95 ("evq->index != rxq->index"));
97 if (rxq->init_state != SFXGE_RXQ_STARTED)
100 expected = rxq->pending++ & (SFXGE_NDESCS - 1);
101 if (id != expected) {
102 evq->exception = B_TRUE;
104 device_printf(sc->dev, "RX completion out of order"
105 " (id=%#x expected=%#x flags=%#x); resetting\n",
106 id, expected, flags);
107 sfxge_schedule_reset(sc);
112 rx_desc = &rxq->queue[id];
114 KASSERT(rx_desc->flags == EFX_DISCARD,
115 ("rx_desc->flags != EFX_DISCARD"));
116 rx_desc->flags = flags;
118 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
119 rx_desc->size = (uint16_t)size;
120 prefetch_read_many(rx_desc->mbuf);
124 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
125 sfxge_ev_qcomplete(evq, B_FALSE);
128 return (evq->rx_done >= SFXGE_EV_BATCH);
132 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
134 struct sfxge_evq *evq;
135 struct sfxge_softc *sc;
137 evq = (struct sfxge_evq *)arg;
140 evq->exception = B_TRUE;
142 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
143 device_printf(sc->dev,
144 "hardware exception (code=%u); resetting\n",
146 sfxge_schedule_reset(sc);
153 sfxge_ev_rxq_flush_done(void *arg, uint32_t label)
155 struct sfxge_evq *evq;
156 struct sfxge_softc *sc;
157 struct sfxge_rxq *rxq;
161 evq = (struct sfxge_evq *)arg;
163 rxq = sc->rxq[label];
165 KASSERT(rxq != NULL, ("rxq == NULL"));
167 /* Resend a software event on the correct queue */
169 evq = sc->evq[index];
171 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
172 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level"));
173 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
175 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
176 ("evq not started"));
177 efx_ev_qpost(evq->common, magic);
183 sfxge_ev_rxq_flush_failed(void *arg, uint32_t label)
185 struct sfxge_evq *evq;
186 struct sfxge_softc *sc;
187 struct sfxge_rxq *rxq;
191 evq = (struct sfxge_evq *)arg;
193 rxq = sc->rxq[label];
195 KASSERT(rxq != NULL, ("rxq == NULL"));
197 /* Resend a software event on the correct queue */
199 evq = sc->evq[index];
201 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
202 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
203 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
205 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
206 ("evq not started"));
207 efx_ev_qpost(evq->common, magic);
213 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
215 struct sfxge_evq *evq;
216 struct sfxge_softc *sc;
217 struct sfxge_txq *txq;
221 evq = (struct sfxge_evq *)arg;
223 txq = sc->txq[label];
225 KASSERT(txq != NULL, ("txq == NULL"));
226 KASSERT(evq->index == txq->evq_index,
227 ("evq->index != txq->evq_index"));
229 if (txq->init_state != SFXGE_TXQ_STARTED)
232 stop = (id + 1) & (SFXGE_NDESCS - 1);
233 id = txq->pending & (SFXGE_NDESCS - 1);
235 delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop);
236 txq->pending += delta;
240 if (txq->next == NULL &&
241 evq->txqs != &(txq->next)) {
243 evq->txqs = &(txq->next);
246 if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
247 sfxge_tx_qcomplete(txq);
250 return (evq->tx_done >= SFXGE_EV_BATCH);
254 sfxge_ev_txq_flush_done(void *arg, uint32_t label)
256 struct sfxge_evq *evq;
257 struct sfxge_softc *sc;
258 struct sfxge_txq *txq;
261 evq = (struct sfxge_evq *)arg;
263 txq = sc->txq[label];
265 KASSERT(txq != NULL, ("txq == NULL"));
266 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
267 ("txq not initialized"));
269 /* Resend a software event on the correct queue */
270 evq = sc->evq[txq->evq_index];
272 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
273 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
274 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
276 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
277 ("evq not started"));
278 efx_ev_qpost(evq->common, magic);
284 sfxge_ev_software(void *arg, uint16_t magic)
286 struct sfxge_evq *evq;
287 struct sfxge_softc *sc;
290 evq = (struct sfxge_evq *)arg;
293 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
294 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
297 case SFXGE_MAGIC_RX_QFLUSH_DONE: {
298 struct sfxge_rxq *rxq = sc->rxq[label];
300 KASSERT(rxq != NULL, ("rxq == NULL"));
301 KASSERT(evq->index == rxq->index,
302 ("evq->index != rxq->index"));
304 sfxge_rx_qflush_done(rxq);
307 case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
308 struct sfxge_rxq *rxq = sc->rxq[label];
310 KASSERT(rxq != NULL, ("rxq == NULL"));
311 KASSERT(evq->index == rxq->index,
312 ("evq->index != rxq->index"));
314 sfxge_rx_qflush_failed(rxq);
317 case SFXGE_MAGIC_RX_QREFILL: {
318 struct sfxge_rxq *rxq = sc->rxq[label];
320 KASSERT(rxq != NULL, ("rxq == NULL"));
321 KASSERT(evq->index == rxq->index,
322 ("evq->index != rxq->index"));
324 sfxge_rx_qrefill(rxq);
327 case SFXGE_MAGIC_TX_QFLUSH_DONE: {
328 struct sfxge_txq *txq = sc->txq[label];
330 KASSERT(txq != NULL, ("txq == NULL"));
331 KASSERT(evq->index == txq->evq_index,
332 ("evq->index != txq->evq_index"));
334 sfxge_tx_qflush_done(txq);
345 sfxge_ev_sram(void *arg, uint32_t code)
351 case EFX_SRAM_UPDATE:
352 EFSYS_PROBE(sram_update);
356 EFSYS_PROBE(sram_clear);
359 case EFX_SRAM_ILLEGAL_CLEAR:
360 EFSYS_PROBE(sram_illegal_clear);
364 KASSERT(B_FALSE, ("Impossible SRAM event"));
372 sfxge_ev_timer(void *arg, uint32_t index)
381 sfxge_ev_wake_up(void *arg, uint32_t index)
390 sfxge_ev_stat_update(struct sfxge_softc *sc)
392 struct sfxge_evq *evq;
396 sx_xlock(&sc->softc_lock);
398 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED)
402 if (now - sc->ev_stats_update_time < hz)
405 sc->ev_stats_update_time = now;
407 /* Add event counts from each event queue in turn */
408 for (index = 0; index < sc->intr.n_alloc; index++) {
409 evq = sc->evq[index];
410 mtx_lock(&evq->lock);
411 efx_ev_qstats_update(evq->common, sc->ev_stats);
412 mtx_unlock(&evq->lock);
415 sx_xunlock(&sc->softc_lock);
419 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
421 struct sfxge_softc *sc = arg1;
422 unsigned int id = arg2;
424 sfxge_ev_stat_update(sc);
426 return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]));
430 sfxge_ev_stat_init(struct sfxge_softc *sc)
432 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
433 struct sysctl_oid_list *stat_list;
437 stat_list = SYSCTL_CHILDREN(sc->stats_node);
439 for (id = 0; id < EV_NQSTATS; id++) {
440 snprintf(name, sizeof(name), "ev_%s",
441 efx_ev_qstat_name(sc->enp, id));
444 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD,
445 sc, id, sfxge_ev_stat_handler, "Q",
451 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
453 struct sfxge_evq *evq;
459 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
460 ("evq->init_state != SFXGE_EVQ_STARTED"));
462 (void)efx_ev_qmoderate(eep, us);
466 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
468 struct sfxge_softc *sc = arg1;
469 struct sfxge_intr *intr = &sc->intr;
470 unsigned int moderation;
474 sx_xlock(&sc->softc_lock);
477 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
481 /* We may not be calling efx_ev_qmoderate() now,
482 * so we have to range-check the value ourselves.
485 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
490 sc->ev_moderation = moderation;
491 if (intr->state == SFXGE_INTR_STARTED) {
492 for (index = 0; index < intr->n_alloc; index++)
493 sfxge_ev_qmoderate(sc, index, moderation);
496 error = SYSCTL_OUT(req, &sc->ev_moderation,
497 sizeof(sc->ev_moderation));
501 sx_xunlock(&sc->softc_lock);
507 sfxge_ev_initialized(void *arg)
509 struct sfxge_evq *evq;
511 evq = (struct sfxge_evq *)arg;
513 KASSERT(evq->init_state == SFXGE_EVQ_STARTING,
514 ("evq not starting"));
516 evq->init_state = SFXGE_EVQ_STARTED;
522 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
524 struct sfxge_evq *evq;
525 struct sfxge_softc *sc;
527 evq = (struct sfxge_evq *)arg;
530 sfxge_mac_link_update(sc, link_mode);
535 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
536 .eec_initialized = sfxge_ev_initialized,
537 .eec_rx = sfxge_ev_rx,
538 .eec_tx = sfxge_ev_tx,
539 .eec_exception = sfxge_ev_exception,
540 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done,
541 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed,
542 .eec_txq_flush_done = sfxge_ev_txq_flush_done,
543 .eec_software = sfxge_ev_software,
544 .eec_sram = sfxge_ev_sram,
545 .eec_wake_up = sfxge_ev_wake_up,
546 .eec_timer = sfxge_ev_timer,
547 .eec_link_change = sfxge_ev_link_change,
552 sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index)
554 struct sfxge_evq *evq;
557 evq = sc->evq[index];
559 mtx_lock(&evq->lock);
561 if (evq->init_state != SFXGE_EVQ_STARTING &&
562 evq->init_state != SFXGE_EVQ_STARTED) {
567 /* Synchronize the DMA memory for reading */
568 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
569 BUS_DMASYNC_POSTREAD);
571 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
572 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
573 KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
574 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
577 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
582 /* Perform any pending completion processing */
583 sfxge_ev_qcomplete(evq, B_TRUE);
585 /* Re-prime the event queue for interrupts */
586 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
589 mtx_unlock(&evq->lock);
594 mtx_unlock(&(evq->lock));
599 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
601 struct sfxge_evq *evq;
603 evq = sc->evq[index];
605 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
606 ("evq->init_state != SFXGE_EVQ_STARTED"));
608 mtx_lock(&evq->lock);
609 evq->init_state = SFXGE_EVQ_INITIALIZED;
611 evq->exception = B_FALSE;
613 /* Add event counts before discarding the common evq state */
614 efx_ev_qstats_update(evq->common, sc->ev_stats);
616 efx_ev_qdestroy(evq->common);
617 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
618 EFX_EVQ_NBUFS(SFXGE_NEVS));
619 mtx_unlock(&evq->lock);
623 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
625 struct sfxge_evq *evq;
630 evq = sc->evq[index];
633 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
634 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
636 /* Clear all events. */
637 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS));
639 /* Program the buffer table. */
640 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
641 EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0)
644 /* Create the common code event queue. */
645 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS,
646 evq->buf_base_id, &evq->common)) != 0)
649 mtx_lock(&evq->lock);
651 /* Set the default moderation */
652 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation);
654 /* Prime the event queue for interrupts */
655 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
658 evq->init_state = SFXGE_EVQ_STARTING;
660 mtx_unlock(&evq->lock);
662 /* Wait for the initialization event */
665 /* Pause for 100 ms */
666 pause("sfxge evq init", hz / 10);
668 /* Check to see if the test event has been processed */
669 if (evq->init_state == SFXGE_EVQ_STARTED)
672 } while (++count < 20);
681 mtx_lock(&evq->lock);
682 evq->init_state = SFXGE_EVQ_INITIALIZED;
684 mtx_unlock(&evq->lock);
685 efx_ev_qdestroy(evq->common);
687 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
688 EFX_EVQ_NBUFS(SFXGE_NEVS));
694 sfxge_ev_stop(struct sfxge_softc *sc)
696 struct sfxge_intr *intr;
703 KASSERT(intr->state == SFXGE_INTR_STARTED,
704 ("Interrupts not started"));
706 /* Stop the event queue(s) */
707 index = intr->n_alloc;
709 sfxge_ev_qstop(sc, index);
711 /* Tear down the event module */
716 sfxge_ev_start(struct sfxge_softc *sc)
718 struct sfxge_intr *intr;
724 KASSERT(intr->state == SFXGE_INTR_STARTED,
725 ("intr->state != SFXGE_INTR_STARTED"));
727 /* Initialize the event module */
728 if ((rc = efx_ev_init(sc->enp)) != 0)
731 /* Start the event queues */
732 for (index = 0; index < intr->n_alloc; index++) {
733 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
740 /* Stop the event queue(s) */
742 sfxge_ev_qstop(sc, index);
744 /* Tear down the event module */
745 efx_ev_fini(sc->enp);
751 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
753 struct sfxge_evq *evq;
755 evq = sc->evq[index];
757 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
758 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
759 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
761 sfxge_dma_free(&evq->mem);
763 sc->evq[index] = NULL;
765 mtx_destroy(&evq->lock);
771 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
773 struct sfxge_evq *evq;
777 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
779 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
782 sc->evq[index] = evq;
785 /* Initialise TX completion list */
786 evq->txqs = &evq->txq;
788 /* Allocate DMA space. */
789 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0)
792 /* Allocate buffer table entries. */
793 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS),
796 mtx_init(&evq->lock, "evq", NULL, MTX_DEF);
798 evq->init_state = SFXGE_EVQ_INITIALIZED;
804 sfxge_ev_fini(struct sfxge_softc *sc)
806 struct sfxge_intr *intr;
811 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
812 ("intr->state != SFXGE_INTR_INITIALIZED"));
814 sc->ev_moderation = 0;
816 /* Tear down the event queue(s). */
817 index = intr->n_alloc;
819 sfxge_ev_qfini(sc, index);
823 sfxge_ev_init(struct sfxge_softc *sc)
825 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
826 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
827 struct sfxge_intr *intr;
833 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
834 ("intr->state != SFXGE_INTR_INITIALIZED"));
836 /* Set default interrupt moderation; add a sysctl to
837 * read and change it.
839 sc->ev_moderation = 30;
840 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
841 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW,
842 sc, 0, sfxge_int_mod_handler, "IU",
843 "sfxge interrupt moderation (us)");
846 * Initialize the event queue(s) - one per interrupt.
848 for (index = 0; index < intr->n_alloc; index++) {
849 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
853 sfxge_ev_stat_init(sc);
859 sfxge_ev_qfini(sc, index);