2 * Copyright (c) 2010-2016 Solarflare Communications Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright notice,
12 * this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 * The views and conclusions contained in the software and documentation are
30 * those of the authors and should not be interpreted as representing official
31 * policies, either expressed or implied, of the FreeBSD Project.
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
39 #include <sys/param.h>
41 #include <sys/kernel.h>
42 #include <sys/malloc.h>
43 #include <sys/queue.h>
45 #include <sys/syslog.h>
46 #include <sys/taskqueue.h>
48 #include <machine/bus.h>
49 #include <machine/resource.h>
51 #include <dev/pci/pcireg.h>
52 #include <dev/pci/pcivar.h>
55 #include <net/rss_config.h>
58 #include "common/efx.h"
63 sfxge_intr_line_filter(void *arg)
65 struct sfxge_evq *evq;
66 struct sfxge_softc *sc;
68 struct sfxge_intr *intr;
72 evq = (struct sfxge_evq *)arg;
77 KASSERT(intr != NULL, ("intr == NULL"));
78 KASSERT(intr->type == EFX_INTR_LINE,
79 ("intr->type != EFX_INTR_LINE"));
81 if (intr->state != SFXGE_INTR_STARTED)
82 return (FILTER_STRAY);
84 (void)efx_intr_status_line(enp, &fatal, &qmask);
87 (void) efx_intr_disable(enp);
88 (void) efx_intr_fatal(enp);
89 return (FILTER_HANDLED);
94 return (FILTER_SCHEDULE_THREAD);
97 /* SF bug 15783: If the function is not asserting its IRQ and
98 * we read the queue mask on the cycle before a flag is added
99 * to the mask, this inhibits the function from asserting the
100 * IRQ even though we don't see the flag set. To work around
101 * this, we must re-prime all event queues and report the IRQ
102 * as handled when we see a mask of zero. To allow for shared
103 * IRQs, we don't repeat this if we see a mask of zero twice
106 if (intr->zero_count++ == 0) {
107 if (evq->init_state == SFXGE_EVQ_STARTED) {
108 if (efx_ev_qpending(evq->common, evq->read_ptr))
109 return (FILTER_SCHEDULE_THREAD);
110 efx_ev_qprime(evq->common, evq->read_ptr);
111 return (FILTER_HANDLED);
115 return (FILTER_STRAY);
119 sfxge_intr_line(void *arg)
121 struct sfxge_evq *evq = arg;
123 (void)sfxge_ev_qpoll(evq);
127 sfxge_intr_message(void *arg)
129 struct sfxge_evq *evq;
130 struct sfxge_softc *sc;
132 struct sfxge_intr *intr;
136 evq = (struct sfxge_evq *)arg;
142 KASSERT(intr != NULL, ("intr == NULL"));
143 KASSERT(intr->type == EFX_INTR_MESSAGE,
144 ("intr->type != EFX_INTR_MESSAGE"));
146 if (__predict_false(intr->state != SFXGE_INTR_STARTED))
149 (void)efx_intr_status_message(enp, index, &fatal);
152 (void)efx_intr_disable(enp);
153 (void)efx_intr_fatal(enp);
157 (void)sfxge_ev_qpoll(evq);
161 sfxge_intr_bus_enable(struct sfxge_softc *sc)
163 struct sfxge_intr *intr;
164 struct sfxge_intr_hdl *table;
165 driver_filter_t *filter;
166 driver_intr_t *handler;
173 switch (intr->type) {
174 case EFX_INTR_MESSAGE:
175 filter = NULL; /* not shared */
176 handler = sfxge_intr_message;
180 filter = sfxge_intr_line_filter;
181 handler = sfxge_intr_line;
185 KASSERT(0, ("Invalid interrupt type"));
189 /* Try to add the handlers */
190 for (index = 0; index < intr->n_alloc; index++) {
191 if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
192 INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
193 sc->evq[index], &table[index].eih_tag)) != 0) {
196 #ifdef SFXGE_HAVE_DESCRIBE_INTR
197 if (intr->n_alloc > 1)
198 bus_describe_intr(sc->dev, table[index].eih_res,
199 table[index].eih_tag, "%d", index);
202 bus_bind_intr(sc->dev, table[index].eih_res,
205 bus_bind_intr(sc->dev, table[index].eih_res, index);
213 /* Remove remaining handlers */
215 bus_teardown_intr(sc->dev, table[index].eih_res,
216 table[index].eih_tag);
222 sfxge_intr_bus_disable(struct sfxge_softc *sc)
224 struct sfxge_intr *intr;
225 struct sfxge_intr_hdl *table;
231 /* Remove all handlers */
232 for (i = 0; i < intr->n_alloc; i++)
233 bus_teardown_intr(sc->dev, table[i].eih_res,
238 sfxge_intr_alloc(struct sfxge_softc *sc, int count)
241 struct sfxge_intr_hdl *table;
242 struct sfxge_intr *intr;
243 struct resource *res;
252 table = malloc(count * sizeof(struct sfxge_intr_hdl),
256 for (i = 0; i < count; i++) {
258 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
259 RF_SHAREABLE | RF_ACTIVE);
261 device_printf(dev, "Couldn't allocate interrupts for "
262 "message %d\n", rid);
266 table[i].eih_rid = rid;
267 table[i].eih_res = res;
272 for (i = 0; i < count; i++)
273 bus_release_resource(dev, SYS_RES_IRQ,
274 table[i].eih_rid, table[i].eih_res);
281 sfxge_intr_teardown_msix(struct sfxge_softc *sc)
284 struct resource *resp;
288 resp = sc->intr.msix_res;
290 rid = rman_get_rid(resp);
291 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
295 sfxge_intr_setup_msix(struct sfxge_softc *sc)
297 struct sfxge_intr *intr;
298 struct resource *resp;
306 /* Check if MSI-X is available. */
307 count = pci_msix_count(dev);
311 /* Do not try to allocate more than already estimated EVQ maximum */
312 KASSERT(sc->evq_max > 0, ("evq_max is zero"));
313 count = MIN(count, sc->evq_max);
316 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
320 if (pci_alloc_msix(dev, &count) != 0) {
321 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
325 /* Allocate interrupt handlers. */
326 if (sfxge_intr_alloc(sc, count) != 0) {
327 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
328 pci_release_msi(dev);
332 intr->type = EFX_INTR_MESSAGE;
333 intr->n_alloc = count;
334 intr->msix_res = resp;
340 sfxge_intr_setup_msi(struct sfxge_softc *sc)
342 struct sfxge_intr_hdl *table;
343 struct sfxge_intr *intr;
353 * Check if MSI is available. All messages must be written to
354 * the same address and on x86 this means the IRQs have the
355 * same CPU affinity. So we only ever allocate 1.
357 count = pci_msi_count(dev) ? 1 : 0;
361 if ((error = pci_alloc_msi(dev, &count)) != 0)
364 /* Allocate interrupt handler. */
365 if (sfxge_intr_alloc(sc, count) != 0) {
366 pci_release_msi(dev);
370 intr->type = EFX_INTR_MESSAGE;
371 intr->n_alloc = count;
377 sfxge_intr_setup_fixed(struct sfxge_softc *sc)
379 struct sfxge_intr_hdl *table;
380 struct sfxge_intr *intr;
381 struct resource *res;
389 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
390 RF_SHAREABLE | RF_ACTIVE);
394 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
395 table[0].eih_rid = rid;
396 table[0].eih_res = res;
398 intr->type = EFX_INTR_LINE;
405 static const char *const __sfxge_err[] = {
407 "SRAM out-of-bounds",
408 "Buffer ID out-of-bounds",
409 "Internal memory parity",
410 "Receive buffer ownership",
411 "Transmit buffer ownership",
412 "Receive descriptor ownership",
413 "Transmit descriptor ownership",
414 "Event queue ownership",
415 "Event queue FIFO overflow",
421 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
424 struct sfxge_softc *sc = (struct sfxge_softc *)arg;
425 device_t dev = sc->dev;
427 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
428 device_get_name(dev), device_get_unit(dev),
429 __sfxge_err[code], dword1, dword0);
433 sfxge_intr_stop(struct sfxge_softc *sc)
435 struct sfxge_intr *intr;
439 KASSERT(intr->state == SFXGE_INTR_STARTED,
440 ("Interrupts not started"));
442 intr->state = SFXGE_INTR_INITIALIZED;
444 /* Disable interrupts at the NIC */
445 efx_intr_disable(sc->enp);
447 /* Disable interrupts at the bus */
448 sfxge_intr_bus_disable(sc);
450 /* Tear down common code interrupt bits. */
451 efx_intr_fini(sc->enp);
455 sfxge_intr_start(struct sfxge_softc *sc)
457 struct sfxge_intr *intr;
462 esmp = &intr->status;
464 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
465 ("Interrupts not initialized"));
467 /* Zero the memory. */
468 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);
470 /* Initialize common code interrupt bits. */
471 (void)efx_intr_init(sc->enp, intr->type, esmp);
473 /* Enable interrupts at the bus */
474 if ((rc = sfxge_intr_bus_enable(sc)) != 0)
477 intr->state = SFXGE_INTR_STARTED;
479 /* Enable interrupts at the NIC */
480 efx_intr_enable(sc->enp);
485 /* Tear down common code interrupt bits. */
486 efx_intr_fini(sc->enp);
488 intr->state = SFXGE_INTR_INITIALIZED;
494 sfxge_intr_fini(struct sfxge_softc *sc)
496 struct sfxge_intr_hdl *table;
497 struct sfxge_intr *intr;
504 esmp = &intr->status;
507 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
508 ("intr->state != SFXGE_INTR_INITIALIZED"));
510 /* Free DMA memory. */
511 sfxge_dma_free(esmp);
513 /* Free interrupt handles. */
514 for (i = 0; i < intr->n_alloc; i++)
515 bus_release_resource(dev, SYS_RES_IRQ,
516 table[i].eih_rid, table[i].eih_res);
518 if (table[0].eih_rid != 0)
519 pci_release_msi(dev);
521 if (intr->msix_res != NULL)
522 sfxge_intr_teardown_msix(sc);
524 /* Free the handle table */
525 free(table, M_SFXGE);
529 /* Clear the interrupt type */
530 intr->type = EFX_INTR_INVALID;
532 intr->state = SFXGE_INTR_UNINITIALIZED;
536 sfxge_intr_init(struct sfxge_softc *sc)
539 struct sfxge_intr *intr;
545 esmp = &intr->status;
547 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
548 ("Interrupts already initialized"));
550 /* Try to setup MSI-X or MSI interrupts if available. */
551 if ((rc = sfxge_intr_setup_msix(sc)) == 0)
552 device_printf(dev, "Using MSI-X interrupts\n");
553 else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
554 device_printf(dev, "Using MSI interrupts\n");
555 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
556 device_printf(dev, "Using fixed interrupts\n");
558 device_printf(dev, "Couldn't setup interrupts\n");
562 /* Set up DMA for interrupts. */
563 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
566 intr->state = SFXGE_INTR_INITIALIZED;