2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2015 Bjoern A. Zeeb
5 * Copyright (c) 2020 Denis Salopek
7 * This software was developed by SRI International and the University of
8 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249
9 * ("MRC2"), as part of the DARPA MRC research programme.
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
38 #include <sys/endian.h>
39 #include <sys/kernel.h>
40 #include <sys/limits.h>
41 #include <sys/module.h>
43 #include <sys/socket.h>
44 #include <sys/sockio.h>
45 #include <sys/sysctl.h>
46 #include <sys/taskqueue.h>
49 #include <net/if_media.h>
50 #include <net/if_types.h>
51 #include <net/if_var.h>
53 #include <netinet/in.h>
54 #include <netinet/if_ether.h>
56 #include <dev/pci/pcivar.h>
57 #include <dev/pci/pcireg.h>
59 #include <machine/bus.h>
63 #define PCI_VENDOR_ID_XILINX 0x10ee
64 #define PCI_DEVICE_ID_SUME 0x7028
66 /* SUME bus driver interface */
67 static int sume_probe(device_t);
68 static int sume_attach(device_t);
69 static int sume_detach(device_t);
71 static device_method_t sume_methods[] = {
72 DEVMETHOD(device_probe, sume_probe),
73 DEVMETHOD(device_attach, sume_attach),
74 DEVMETHOD(device_detach, sume_detach),
78 static driver_t sume_driver = {
81 sizeof(struct sume_adapter)
85 * The DMA engine for SUME generates interrupts for each RX/TX transaction.
86 * Depending on the channel (0 if packet transaction, 1 if register transaction)
87 * the used bits of the interrupt vector will be the lowest or the second lowest
90 * When receiving packets from SUME (RX):
91 * (1) SUME received a packet on one of the interfaces.
92 * (2) SUME generates an interrupt vector, bit 00001 is set (channel 0 - new RX
94 * (3) We read the length of the incoming packet and the offset along with the
95 * 'last' flag from the SUME registers.
96 * (4) We prepare for the DMA transaction by setting the bouncebuffer on the
97 * address buf_addr. For now, this is how it's done:
98 * - First 3*sizeof(uint32_t) bytes are: lower and upper 32 bits of physical
99 * address where we want the data to arrive (buf_addr[0] and buf_addr[1]),
100 * and length of incoming data (buf_addr[2]).
101 * - Data will start right after, at buf_addr+3*sizeof(uint32_t). The
102 * physical address buf_hw_addr is a block of contiguous memory mapped to
103 * buf_addr, so we can set the incoming data's physical address (buf_addr[0]
104 * and buf_addr[1]) to buf_hw_addr+3*sizeof(uint32_t).
105 * (5) We notify SUME that the bouncebuffer is ready for the transaction by
106 * writing the lower/upper physical address buf_hw_addr to the SUME
107 * registers RIFFA_TX_SG_ADDR_LO_REG_OFF and RIFFA_TX_SG_ADDR_HI_REG_OFF as
108 * well as the number of segments to the register RIFFA_TX_SG_LEN_REG_OFF.
109 * (6) SUME generates an interrupt vector, bit 00010 is set (channel 0 -
110 * bouncebuffer received).
111 * (7) SUME generates an interrupt vector, bit 00100 is set (channel 0 -
112 * transaction is done).
113 * (8) SUME can do both steps (6) and (7) using the same interrupt.
114 * (8) We read the first 16 bytes (metadata) of the received data and note the
115 * incoming interface so we can later forward it to the right one in the OS
116 * (sume0, sume1, sume2 or sume3).
117 * (10) We create an mbuf and copy the data from the bouncebuffer to the mbuf
118 * and set the mbuf rcvif to the incoming interface.
119 * (11) We forward the mbuf to the appropriate interface via ifp->if_input.
121 * When sending packets to SUME (TX):
122 * (1) The OS calls sume_if_start() function on TX.
123 * (2) We get the mbuf packet data and copy it to the
124 * buf_addr+3*sizeof(uint32_t) + metadata 16 bytes.
125 * (3) We create the metadata based on the output interface and copy it to the
126 * buf_addr+3*sizeof(uint32_t).
127 * (4) We write the offset/last and length of the packet to the SUME registers
128 * RIFFA_RX_OFFLAST_REG_OFF and RIFFA_RX_LEN_REG_OFF.
129 * (5) We fill the bouncebuffer by filling the first 3*sizeof(uint32_t) bytes
130 * with the physical address and length just as in RX step (4).
131 * (6) We notify SUME that the bouncebuffer is ready by writing to SUME
132 * registers RIFFA_RX_SG_ADDR_LO_REG_OFF, RIFFA_RX_SG_ADDR_HI_REG_OFF and
133 * RIFFA_RX_SG_LEN_REG_OFF just as in RX step (5).
134 * (7) SUME generates an interrupt vector, bit 01000 is set (channel 0 -
135 * bouncebuffer is read).
136 * (8) SUME generates an interrupt vector, bit 10000 is set (channel 0 -
137 * transaction is done).
138 * (9) SUME can do both steps (7) and (8) using the same interrupt.
141 * Every module in the SUME hardware has its own set of internal registers
142 * (IDs, for debugging and statistic purposes, etc.). Their base addresses are
143 * defined in 'projects/reference_nic/hw/tcl/reference_nic_defines.tcl' and the
144 * offsets to different memory locations of every module are defined in their
145 * corresponding folder inside the library. These registers can be RO/RW and
146 * there is a special method to fetch/change this data over 1 or 2 DMA
147 * transactions. For writing, by calling the sume_module_reg_write(). For
148 * reading, by calling the sume_module_reg_write() and then
149 * sume_module_reg_read(). Check those functions for more information.
152 MALLOC_DECLARE(M_SUME);
153 MALLOC_DEFINE(M_SUME, "sume", "NetFPGA SUME device driver");
155 static void check_tx_queues(struct sume_adapter *);
156 static void sume_fill_bb_desc(struct sume_adapter *, struct riffa_chnl_dir *,
159 static struct unrhdr *unr;
165 {PCI_DEVICE_ID_SUME, "NetFPGA SUME reference NIC"},
168 static inline uint32_t
169 read_reg(struct sume_adapter *adapter, int offset)
172 return (bus_space_read_4(adapter->bt, adapter->bh, offset << 2));
176 write_reg(struct sume_adapter *adapter, int offset, uint32_t val)
179 bus_space_write_4(adapter->bt, adapter->bh, offset << 2, val);
183 sume_probe(device_t dev)
186 uint16_t v = pci_get_vendor(dev);
187 uint16_t d = pci_get_device(dev);
189 if (v != PCI_VENDOR_ID_XILINX)
192 for (i = 0; i < nitems(sume_pciids); i++) {
193 if (d == sume_pciids[i].device) {
194 device_set_desc(dev, sume_pciids[i].desc);
195 return (BUS_PROBE_DEFAULT);
203 * Building mbuf for packet received from SUME. We expect to receive 'len'
204 * bytes of data (including metadata) written from the bouncebuffer address
205 * buf_addr+3*sizeof(uint32_t). Metadata will tell us which SUME interface
206 * received the packet (sport will be 1, 2, 4 or 8), the packet length (plen),
207 * and the magic word needs to be 0xcafe. When we have the packet data, we
208 * create an mbuf and copy the data to it using m_copyback() function, set the
209 * correct interface to rcvif and return the mbuf to be later sent to the OS
213 sume_rx_build_mbuf(struct sume_adapter *adapter, uint32_t len)
215 struct nf_priv *nf_priv;
217 struct ifnet *ifp = NULL;
219 uint16_t dport, plen, magic;
220 device_t dev = adapter->dev;
221 uint8_t *indata = (uint8_t *)
222 adapter->recv[SUME_RIFFA_CHANNEL_DATA]->buf_addr +
223 sizeof(struct nf_bb_desc);
224 struct nf_metadata *mdata = (struct nf_metadata *) indata;
226 /* The metadata header is 16 bytes. */
227 if (len < sizeof(struct nf_metadata)) {
228 device_printf(dev, "short frame (%d)\n", len);
229 adapter->packets_err++;
230 adapter->bytes_err += len;
234 dport = le16toh(mdata->dport);
235 plen = le16toh(mdata->plen);
236 magic = le16toh(mdata->magic);
238 if (sizeof(struct nf_metadata) + plen > len ||
239 magic != SUME_RIFFA_MAGIC) {
240 device_printf(dev, "corrupted packet (%zd + %d > %d || magic "
241 "0x%04x != 0x%04x)\n", sizeof(struct nf_metadata), plen,
242 len, magic, SUME_RIFFA_MAGIC);
246 /* We got the packet from one of the even bits */
247 np = (ffs(dport & SUME_DPORT_MASK) >> 1) - 1;
248 if (np > SUME_NPORTS) {
249 device_printf(dev, "invalid destination port 0x%04x (%d)\n",
251 adapter->packets_err++;
252 adapter->bytes_err += plen;
255 ifp = adapter->ifp[np];
256 nf_priv = ifp->if_softc;
257 nf_priv->stats.rx_packets++;
258 nf_priv->stats.rx_bytes += plen;
260 /* If the interface is down, well, we are done. */
261 if (!(ifp->if_flags & IFF_UP)) {
262 nf_priv->stats.ifc_down_packets++;
263 nf_priv->stats.ifc_down_bytes += plen;
267 if (adapter->sume_debug)
268 printf("Building mbuf with length: %d\n", plen);
270 m = m_getm(NULL, plen, M_NOWAIT, MT_DATA);
272 adapter->packets_err++;
273 adapter->bytes_err += plen;
277 /* Copy the data in at the right offset. */
278 m_copyback(m, 0, plen, (void *) (indata + sizeof(struct nf_metadata)));
279 m->m_pkthdr.rcvif = ifp;
285 * SUME interrupt handler for when we get a valid interrupt from the board.
286 * Theoretically, we can receive interrupt for any of the available channels,
287 * but RIFFA DMA uses only 2: 0 and 1, so we use only vect0. The vector is a 32
288 * bit number, using 5 bits for every channel, the least significant bits
289 * correspond to channel 0 and the next 5 bits correspond to channel 1. Vector
290 * bits for RX/TX are:
292 * bit 0 - new transaction from SUME
293 * bit 1 - SUME received our bouncebuffer address
294 * bit 2 - SUME copied the received data to our bouncebuffer, transaction done
296 * bit 3 - SUME received our bouncebuffer address
297 * bit 4 - SUME copied the data from our bouncebuffer, transaction done
299 * There are two finite state machines (one for TX, one for RX). We loop
300 * through channels 0 and 1 to check and our current state and which interrupt
303 * SUME_RIFFA_CHAN_STATE_IDLE: waiting for the first TX transaction.
304 * SUME_RIFFA_CHAN_STATE_READY: we prepared (filled with data) the bouncebuffer
305 * and triggered the SUME for the TX transaction. Waiting for interrupt bit 3
306 * to go to the next state.
307 * SUME_RIFFA_CHAN_STATE_READ: waiting for interrupt bit 4 (for SUME to send
308 * our packet). Then we get the length of the sent data and go back to the
311 * SUME_RIFFA_CHAN_STATE_IDLE: waiting for the interrupt bit 0 (new RX
312 * transaction). When we get it, we prepare our bouncebuffer for reading and
313 * trigger the SUME to start the transaction. Go to the next state.
314 * SUME_RIFFA_CHAN_STATE_READY: waiting for the interrupt bit 1 (SUME got our
315 * bouncebuffer). Go to the next state.
316 * SUME_RIFFA_CHAN_STATE_READ: SUME copied data and our bouncebuffer is ready,
317 * we can build the mbuf and go back to the IDLE state.
320 sume_intr_handler(void *arg)
322 struct sume_adapter *adapter = arg;
323 uint32_t vect, vect0, len;
325 device_t dev = adapter->dev;
326 struct mbuf *m = NULL;
327 struct ifnet *ifp = NULL;
328 struct riffa_chnl_dir *send, *recv;
332 vect0 = read_reg(adapter, RIFFA_IRQ_REG0_OFF);
333 if ((vect0 & SUME_INVALID_VECT) != 0) {
334 SUME_UNLOCK(adapter);
339 * We only have one interrupt for all channels and no way
340 * to quickly lookup for which channel(s) we got an interrupt?
342 for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) {
343 vect = vect0 >> (5 * ch);
344 send = adapter->send[ch];
345 recv = adapter->recv[ch];
348 while ((vect & (SUME_MSI_TXBUF | SUME_MSI_TXDONE)) &&
350 if (adapter->sume_debug)
351 device_printf(dev, "TX ch %d state %u vect = "
352 "0x%08x\n", ch, send->state, vect);
353 switch (send->state) {
354 case SUME_RIFFA_CHAN_STATE_IDLE:
356 case SUME_RIFFA_CHAN_STATE_READY:
357 if (!(vect & SUME_MSI_TXBUF)) {
358 device_printf(dev, "ch %d unexpected "
359 "interrupt in send+3 state %u: "
360 "vect = 0x%08x\n", ch, send->state,
365 send->state = SUME_RIFFA_CHAN_STATE_READ;
366 vect &= ~SUME_MSI_TXBUF;
368 case SUME_RIFFA_CHAN_STATE_READ:
369 if (!(vect & SUME_MSI_TXDONE)) {
370 device_printf(dev, "ch %d unexpected "
371 "interrupt in send+4 state %u: "
372 "vect = 0x%08x\n", ch, send->state,
377 send->state = SUME_RIFFA_CHAN_STATE_LEN;
379 len = read_reg(adapter, RIFFA_CHNL_REG(ch,
380 RIFFA_RX_TNFR_LEN_REG_OFF));
381 if (ch == SUME_RIFFA_CHANNEL_DATA) {
383 SUME_RIFFA_CHAN_STATE_IDLE;
384 check_tx_queues(adapter);
385 } else if (ch == SUME_RIFFA_CHANNEL_REG)
386 wakeup(&send->event);
388 device_printf(dev, "ch %d unexpected "
389 "interrupt in send+4 state %u: "
390 "vect = 0x%08x\n", ch, send->state,
394 vect &= ~SUME_MSI_TXDONE;
396 case SUME_RIFFA_CHAN_STATE_LEN:
399 device_printf(dev, "unknown TX state!\n");
404 if ((vect & (SUME_MSI_TXBUF | SUME_MSI_TXDONE)) &&
406 device_printf(dev, "ch %d ignoring vect = 0x%08x "
407 "during TX; not in recovery; state = %d loops = "
408 "%d\n", ch, vect, send->state, loops);
411 while ((vect & (SUME_MSI_RXQUE | SUME_MSI_RXBUF |
412 SUME_MSI_RXDONE)) && loops < 5) {
413 if (adapter->sume_debug)
414 device_printf(dev, "RX ch %d state %u vect = "
415 "0x%08x\n", ch, recv->state, vect);
416 switch (recv->state) {
417 case SUME_RIFFA_CHAN_STATE_IDLE:
418 if (!(vect & SUME_MSI_RXQUE)) {
419 device_printf(dev, "ch %d unexpected "
420 "interrupt in recv+0 state %u: "
421 "vect = 0x%08x\n", ch, recv->state,
428 /* Clear recovery state. */
431 /* Get offset and length. */
432 recv->offlast = read_reg(adapter,
434 RIFFA_TX_OFFLAST_REG_OFF));
435 recv->len = read_reg(adapter, RIFFA_CHNL_REG(ch,
436 RIFFA_TX_LEN_REG_OFF));
438 /* Boundary checks. */
439 max_ptr = (uint32_t)((uintptr_t)recv->buf_addr
440 + SUME_RIFFA_OFFSET(recv->offlast)
441 + SUME_RIFFA_LEN(recv->len) - 1);
443 (uint32_t)((uintptr_t)recv->buf_addr))
444 device_printf(dev, "receive buffer "
445 "wrap-around overflow.\n");
446 if (SUME_RIFFA_OFFSET(recv->offlast) +
447 SUME_RIFFA_LEN(recv->len) >
448 adapter->sg_buf_size)
449 device_printf(dev, "receive buffer too"
452 /* Fill the bouncebuf "descriptor". */
453 sume_fill_bb_desc(adapter, recv,
454 SUME_RIFFA_LEN(recv->len));
456 bus_dmamap_sync(recv->ch_tag, recv->ch_map,
457 BUS_DMASYNC_PREREAD |
458 BUS_DMASYNC_PREWRITE);
459 write_reg(adapter, RIFFA_CHNL_REG(ch,
460 RIFFA_TX_SG_ADDR_LO_REG_OFF),
461 SUME_RIFFA_LO_ADDR(recv->buf_hw_addr));
462 write_reg(adapter, RIFFA_CHNL_REG(ch,
463 RIFFA_TX_SG_ADDR_HI_REG_OFF),
464 SUME_RIFFA_HI_ADDR(recv->buf_hw_addr));
465 write_reg(adapter, RIFFA_CHNL_REG(ch,
466 RIFFA_TX_SG_LEN_REG_OFF),
468 bus_dmamap_sync(recv->ch_tag, recv->ch_map,
469 BUS_DMASYNC_POSTREAD |
470 BUS_DMASYNC_POSTWRITE);
472 recv->state = SUME_RIFFA_CHAN_STATE_READY;
473 vect &= ~SUME_MSI_RXQUE;
475 case SUME_RIFFA_CHAN_STATE_READY:
476 if (!(vect & SUME_MSI_RXBUF)) {
477 device_printf(dev, "ch %d unexpected "
478 "interrupt in recv+1 state %u: "
479 "vect = 0x%08x\n", ch, recv->state,
484 recv->state = SUME_RIFFA_CHAN_STATE_READ;
485 vect &= ~SUME_MSI_RXBUF;
487 case SUME_RIFFA_CHAN_STATE_READ:
488 if (!(vect & SUME_MSI_RXDONE)) {
489 device_printf(dev, "ch %d unexpected "
490 "interrupt in recv+2 state %u: "
491 "vect = 0x%08x\n", ch, recv->state,
496 len = read_reg(adapter, RIFFA_CHNL_REG(ch,
497 RIFFA_TX_TNFR_LEN_REG_OFF));
499 /* Remember, len and recv->len are words. */
500 if (ch == SUME_RIFFA_CHANNEL_DATA) {
501 m = sume_rx_build_mbuf(adapter,
504 SUME_RIFFA_CHAN_STATE_IDLE;
505 } else if (ch == SUME_RIFFA_CHANNEL_REG)
506 wakeup(&recv->event);
508 device_printf(dev, "ch %d unexpected "
509 "interrupt in recv+2 state %u: "
510 "vect = 0x%08x\n", ch, recv->state,
514 vect &= ~SUME_MSI_RXDONE;
516 case SUME_RIFFA_CHAN_STATE_LEN:
519 device_printf(dev, "unknown RX state!\n");
524 if ((vect & (SUME_MSI_RXQUE | SUME_MSI_RXBUF |
525 SUME_MSI_RXDONE)) && recv->recovery) {
526 device_printf(dev, "ch %d ignoring vect = 0x%08x "
527 "during RX; not in recovery; state = %d, loops = "
528 "%d\n", ch, vect, recv->state, loops);
530 /* Clean the unfinished transaction. */
531 if (ch == SUME_RIFFA_CHANNEL_REG &&
532 vect & SUME_MSI_RXDONE) {
533 read_reg(adapter, RIFFA_CHNL_REG(ch,
534 RIFFA_TX_TNFR_LEN_REG_OFF));
539 SUME_UNLOCK(adapter);
542 ifp = m->m_pkthdr.rcvif;
543 (*ifp->if_input)(ifp, m);
548 * As we cannot disable interrupt generation, ignore early interrupts by waiting
549 * for the adapter to go into the 'running' state.
552 sume_intr_filter(void *arg)
554 struct sume_adapter *adapter = arg;
556 if (adapter->running == 0)
557 return (FILTER_STRAY);
559 return (FILTER_SCHEDULE_THREAD);
563 sume_probe_riffa_pci(struct sume_adapter *adapter)
565 device_t dev = adapter->dev;
566 int error, count, capmem;
567 uint32_t reg, devctl, linkctl;
569 pci_enable_busmaster(dev);
571 adapter->rid = PCIR_BAR(0);
572 adapter->bar0_addr = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
573 &adapter->rid, RF_ACTIVE);
574 if (adapter->bar0_addr == NULL) {
575 device_printf(dev, "unable to allocate bus resource: "
579 adapter->bt = rman_get_bustag(adapter->bar0_addr);
580 adapter->bh = rman_get_bushandle(adapter->bar0_addr);
581 adapter->bar0_len = rman_get_size(adapter->bar0_addr);
582 if (adapter->bar0_len != 1024) {
583 device_printf(dev, "BAR0 resource length %lu != 1024\n",
588 count = pci_msi_count(dev);
589 error = pci_alloc_msi(dev, &count);
591 device_printf(dev, "unable to allocate bus resource: PCI "
596 adapter->irq.rid = 1; /* Should be 1, thus says pci_alloc_msi() */
597 adapter->irq.res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
598 &adapter->irq.rid, RF_SHAREABLE | RF_ACTIVE);
599 if (adapter->irq.res == NULL) {
600 device_printf(dev, "unable to allocate bus resource: IRQ "
605 error = bus_setup_intr(dev, adapter->irq.res, INTR_MPSAFE |
606 INTR_TYPE_NET, sume_intr_filter, sume_intr_handler, adapter,
609 device_printf(dev, "failed to setup interrupt for rid %d, name"
610 " %s: %d\n", adapter->irq.rid, "SUME_INTR", error);
614 if (pci_find_cap(dev, PCIY_EXPRESS, &capmem) != 0) {
615 device_printf(dev, "PCI not PCIe capable\n");
619 devctl = pci_read_config(dev, capmem + PCIER_DEVICE_CTL, 2);
620 pci_write_config(dev, capmem + PCIER_DEVICE_CTL, (devctl |
621 PCIEM_CTL_EXT_TAG_FIELD), 2);
623 devctl = pci_read_config(dev, capmem + PCIER_DEVICE_CTL2, 2);
624 pci_write_config(dev, capmem + PCIER_DEVICE_CTL2, (devctl |
625 PCIEM_CTL2_ID_ORDERED_REQ_EN), 2);
627 linkctl = pci_read_config(dev, capmem + PCIER_LINK_CTL, 2);
628 pci_write_config(dev, capmem + PCIER_LINK_CTL, (linkctl |
629 PCIEM_LINK_CTL_RCB), 2);
631 reg = read_reg(adapter, RIFFA_INFO_REG_OFF);
632 adapter->num_sg = RIFFA_SG_ELEMS * ((reg >> 19) & 0xf);
633 adapter->sg_buf_size = RIFFA_SG_BUF_SIZE * ((reg >> 19) & 0xf);
636 /* Check bus master is enabled. */
637 if (((reg >> 4) & 0x1) != 1) {
638 device_printf(dev, "bus master not enabled: %d\n",
642 /* Check link parameters are valid. */
643 if (((reg >> 5) & 0x3f) == 0 || ((reg >> 11) & 0x3) == 0) {
644 device_printf(dev, "link parameters not valid: %d %d\n",
645 (reg >> 5) & 0x3f, (reg >> 11) & 0x3);
648 /* Check # of channels are within valid range. */
649 if ((reg & 0xf) == 0 || (reg & 0xf) > RIFFA_MAX_CHNLS) {
650 device_printf(dev, "number of channels out of range: %d\n",
654 /* Check bus width. */
655 if (((reg >> 19) & 0xf) == 0 ||
656 ((reg >> 19) & 0xf) > RIFFA_MAX_BUS_WIDTH_PARAM) {
657 device_printf(dev, "bus width out of range: %d\n",
662 device_printf(dev, "[riffa] # of channels: %d\n",
664 device_printf(dev, "[riffa] bus interface width: %d\n",
665 ((reg >> 19) & 0xf) << 5);
666 device_printf(dev, "[riffa] bus master enabled: %d\n",
668 device_printf(dev, "[riffa] negotiated link width: %d\n",
670 device_printf(dev, "[riffa] negotiated rate width: %d MTs\n",
671 ((reg >> 11) & 0x3) * 2500);
672 device_printf(dev, "[riffa] max downstream payload: %d B\n",
673 128 << ((reg >> 13) & 0x7));
674 device_printf(dev, "[riffa] max upstream payload: %d B\n",
675 128 << ((reg >> 16) & 0x7));
680 /* If there is no sume_if_init, the ether_ioctl panics. */
682 sume_if_init(void *sc)
686 /* Write the address and length for our incoming / outgoing transaction. */
688 sume_fill_bb_desc(struct sume_adapter *adapter, struct riffa_chnl_dir *p,
691 struct nf_bb_desc *bouncebuf = (struct nf_bb_desc *) p->buf_addr;
693 bouncebuf->lower = (p->buf_hw_addr + sizeof(struct nf_bb_desc));
694 bouncebuf->upper = (p->buf_hw_addr + sizeof(struct nf_bb_desc)) >> 32;
695 bouncebuf->len = len >> 2;
698 /* Module register locked write. */
700 sume_modreg_write_locked(struct sume_adapter *adapter)
702 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG];
704 /* Let the FPGA know about the transfer. */
705 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG,
706 RIFFA_RX_OFFLAST_REG_OFF), SUME_OFFLAST);
707 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG,
708 RIFFA_RX_LEN_REG_OFF), send->len); /* words */
710 /* Fill the bouncebuf "descriptor". */
711 sume_fill_bb_desc(adapter, send, SUME_RIFFA_LEN(send->len));
713 /* Update the state before intiating the DMA to avoid races. */
714 send->state = SUME_RIFFA_CHAN_STATE_READY;
716 bus_dmamap_sync(send->ch_tag, send->ch_map,
717 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
719 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG,
720 RIFFA_RX_SG_ADDR_LO_REG_OFF),
721 SUME_RIFFA_LO_ADDR(send->buf_hw_addr));
722 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG,
723 RIFFA_RX_SG_ADDR_HI_REG_OFF),
724 SUME_RIFFA_HI_ADDR(send->buf_hw_addr));
725 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG,
726 RIFFA_RX_SG_LEN_REG_OFF), 4 * send->num_sg);
727 bus_dmamap_sync(send->ch_tag, send->ch_map,
728 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
734 * Request a register read or write (depending on optype).
735 * If optype is set (0x1f) this will result in a register write,
736 * otherwise this will result in a register read request at the given
737 * address and the result will need to be DMAed back.
740 sume_module_reg_write(struct nf_priv *nf_priv, struct sume_ifreq *sifr,
743 struct sume_adapter *adapter = nf_priv->adapter;
744 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG];
745 struct nf_regop_data *data;
749 * 1. Make sure the channel is free; otherwise return EBUSY.
750 * 2. Prepare the memory in the bounce buffer (which we always
752 * 3. Start the DMA process.
753 * 4. Sleep and wait for result and return success or error.
757 if (send->state != SUME_RIFFA_CHAN_STATE_IDLE) {
758 SUME_UNLOCK(adapter);
762 data = (struct nf_regop_data *) (send->buf_addr +
763 sizeof(struct nf_bb_desc));
764 data->addr = htole32(sifr->addr);
765 data->val = htole32(sifr->val);
766 /* Tag to indentify request. */
767 data->rtag = htole32(++send->rtag);
768 data->optype = htole32(optype);
769 send->len = sizeof(struct nf_regop_data) / 4; /* words */
771 error = sume_modreg_write_locked(adapter);
773 SUME_UNLOCK(adapter);
777 /* Timeout after 1s. */
778 if (send->state != SUME_RIFFA_CHAN_STATE_LEN)
779 error = msleep(&send->event, &adapter->lock, 0,
780 "Waiting recv finish", 1 * hz);
782 /* This was a write so we are done; were interrupted, or timed out. */
783 if (optype != SUME_MR_READ || error != 0 || error == EWOULDBLOCK) {
784 send->state = SUME_RIFFA_CHAN_STATE_IDLE;
785 if (optype == SUME_MR_READ)
793 * For read requests we will update state once we are done
794 * having read the result to avoid any two outstanding
795 * transactions, or we need a queue and validate tags,
796 * which is a lot of work for a low priority, infrequent
800 SUME_UNLOCK(adapter);
805 /* Module register read. */
807 sume_module_reg_read(struct nf_priv *nf_priv, struct sume_ifreq *sifr)
809 struct sume_adapter *adapter = nf_priv->adapter;
810 struct riffa_chnl_dir *recv = adapter->recv[SUME_RIFFA_CHANNEL_REG];
811 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG];
812 struct nf_regop_data *data;
816 * 0. Sleep waiting for result if needed (unless condition is
818 * 1. Read DMA results.
819 * 2. Update state on *TX* to IDLE to allow next read to start.
823 bus_dmamap_sync(recv->ch_tag, recv->ch_map,
824 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
826 * We only need to be woken up at the end of the transaction.
829 if (recv->state != SUME_RIFFA_CHAN_STATE_READ)
830 error = msleep(&recv->event, &adapter->lock, 0,
831 "Waiting transaction finish", 1 * hz);
833 if (recv->state != SUME_RIFFA_CHAN_STATE_READ || error == EWOULDBLOCK) {
834 SUME_UNLOCK(adapter);
835 device_printf(adapter->dev, "wait error: %d\n", error);
836 return (EWOULDBLOCK);
839 bus_dmamap_sync(recv->ch_tag, recv->ch_map,
840 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
843 * Read reply data and validate address and tag.
844 * Note: we do access the send side without lock but the state
845 * machine does prevent the data from changing.
847 data = (struct nf_regop_data *) (recv->buf_addr +
848 sizeof(struct nf_bb_desc));
850 if (le32toh(data->rtag) != send->rtag)
851 device_printf(adapter->dev, "rtag error: 0x%08x 0x%08x\n",
852 le32toh(data->rtag), send->rtag);
854 sifr->val = le32toh(data->val);
855 recv->state = SUME_RIFFA_CHAN_STATE_IDLE;
858 send->state = SUME_RIFFA_CHAN_STATE_IDLE;
860 SUME_UNLOCK(adapter);
865 /* Read value from a module register and return it to a sume_ifreq. */
867 get_modreg_value(struct nf_priv *nf_priv, struct sume_ifreq *sifr)
871 error = sume_module_reg_write(nf_priv, sifr, SUME_MR_READ);
873 error = sume_module_reg_read(nf_priv, sifr);
879 sume_if_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
881 struct ifreq *ifr = (struct ifreq *) data;
882 struct nf_priv *nf_priv = ifp->if_softc;
883 struct sume_ifreq sifr;
889 error = ifmedia_ioctl(ifp, ifr, &nf_priv->media, cmd);
892 case SUME_IOCTL_CMD_WRITE_REG:
893 error = copyin(ifr_data_get_ptr(ifr), &sifr, sizeof(sifr));
898 error = sume_module_reg_write(nf_priv, &sifr, SUME_MR_WRITE);
901 case SUME_IOCTL_CMD_READ_REG:
902 error = copyin(ifr_data_get_ptr(ifr), &sifr, sizeof(sifr));
908 error = get_modreg_value(nf_priv, &sifr);
912 error = copyout(&sifr, ifr_data_get_ptr(ifr), sizeof(sifr));
919 /* Silence tcpdump 'promisc mode not supported' warning. */
920 if (ifp->if_flags & IFF_PROMISC)
924 error = ether_ioctl(ifp, cmd, data);
932 sume_media_change(struct ifnet *ifp)
934 struct nf_priv *nf_priv = ifp->if_softc;
935 struct ifmedia *ifm = &nf_priv->media;
937 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
940 if (IFM_SUBTYPE(ifm->ifm_media) == IFM_10G_SR)
941 ifp->if_baudrate = ifmedia_baudrate(IFM_ETHER | IFM_10G_SR);
943 ifp->if_baudrate = ifmedia_baudrate(ifm->ifm_media);
949 sume_update_link_status(struct ifnet *ifp)
951 struct nf_priv *nf_priv = ifp->if_softc;
952 struct sume_adapter *adapter = nf_priv->adapter;
953 struct sume_ifreq sifr;
956 sifr.addr = SUME_STATUS_ADDR(nf_priv->port);
959 if (get_modreg_value(nf_priv, &sifr))
962 link_status = SUME_LINK_STATUS(sifr.val);
964 if (!link_status && nf_priv->link_up) {
965 if_link_state_change(ifp, LINK_STATE_DOWN);
966 nf_priv->link_up = 0;
967 if (adapter->sume_debug)
968 device_printf(adapter->dev, "port %d link state "
969 "changed to DOWN\n", nf_priv->unit);
970 } else if (link_status && !nf_priv->link_up) {
971 nf_priv->link_up = 1;
972 if_link_state_change(ifp, LINK_STATE_UP);
973 if (adapter->sume_debug)
974 device_printf(adapter->dev, "port %d link state "
975 "changed to UP\n", nf_priv->unit);
980 sume_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
982 struct nf_priv *nf_priv = ifp->if_softc;
983 struct ifmedia *ifm = &nf_priv->media;
985 if (ifm->ifm_cur->ifm_media == (IFM_ETHER | IFM_10G_SR) &&
986 (ifp->if_flags & IFF_UP))
987 ifmr->ifm_active = IFM_ETHER | IFM_10G_SR;
989 ifmr->ifm_active = ifm->ifm_cur->ifm_media;
991 ifmr->ifm_status |= IFM_AVALID;
993 sume_update_link_status(ifp);
995 if (nf_priv->link_up)
996 ifmr->ifm_status |= IFM_ACTIVE;
1000 * Packet to transmit. We take the packet data from the mbuf and copy it to the
1001 * bouncebuffer address buf_addr+3*sizeof(uint32_t)+16. The 16 bytes before the
1002 * packet data are for metadata: sport/dport (depending on our source
1003 * interface), packet length and magic 0xcafe. We tell the SUME about the
1004 * transfer, fill the first 3*sizeof(uint32_t) bytes of the bouncebuffer with
1005 * the information about the start and length of the packet and trigger the
1009 sume_if_start_locked(struct ifnet *ifp)
1012 struct nf_priv *nf_priv = ifp->if_softc;
1013 struct sume_adapter *adapter = nf_priv->adapter;
1014 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_DATA];
1016 struct nf_metadata *mdata;
1017 int plen = SUME_MIN_PKT_SIZE;
1019 KASSERT(mtx_owned(&adapter->lock), ("SUME lock not owned"));
1020 KASSERT(send->state == SUME_RIFFA_CHAN_STATE_IDLE,
1021 ("SUME not in IDLE state"));
1023 IFQ_DEQUEUE(&ifp->if_snd, m);
1027 /* Packets large enough do not need to be padded */
1028 if (m->m_pkthdr.len > SUME_MIN_PKT_SIZE)
1029 plen = m->m_pkthdr.len;
1031 if (adapter->sume_debug)
1032 device_printf(adapter->dev, "sending %d bytes to %s%d\n", plen,
1033 SUME_ETH_DEVICE_NAME, nf_priv->unit);
1035 outbuf = (uint8_t *) send->buf_addr + sizeof(struct nf_bb_desc);
1036 mdata = (struct nf_metadata *) outbuf;
1038 /* Clear the recovery flag. */
1041 /* Make sure we fit with the 16 bytes nf_metadata. */
1042 if (m->m_pkthdr.len + sizeof(struct nf_metadata) >
1043 adapter->sg_buf_size) {
1044 device_printf(adapter->dev, "packet too big for bounce buffer "
1045 "(%d)\n", m->m_pkthdr.len);
1047 nf_priv->stats.tx_dropped++;
1051 bus_dmamap_sync(send->ch_tag, send->ch_map,
1052 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1054 /* Zero out the padded data */
1055 if (m->m_pkthdr.len < SUME_MIN_PKT_SIZE)
1056 bzero(outbuf + sizeof(struct nf_metadata), SUME_MIN_PKT_SIZE);
1057 /* Skip the first 16 bytes for the metadata. */
1058 m_copydata(m, 0, m->m_pkthdr.len, outbuf + sizeof(struct nf_metadata));
1059 send->len = (sizeof(struct nf_metadata) + plen + 3) / 4;
1061 /* Fill in the metadata: CPU(DMA) ports are odd, MAC ports are even. */
1062 mdata->sport = htole16(1 << (nf_priv->port * 2 + 1));
1063 mdata->dport = htole16(1 << (nf_priv->port * 2));
1064 mdata->plen = htole16(plen);
1065 mdata->magic = htole16(SUME_RIFFA_MAGIC);
1066 mdata->t1 = htole32(0);
1067 mdata->t2 = htole32(0);
1069 /* Let the FPGA know about the transfer. */
1070 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA,
1071 RIFFA_RX_OFFLAST_REG_OFF), SUME_OFFLAST);
1072 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA,
1073 RIFFA_RX_LEN_REG_OFF), send->len);
1075 /* Fill the bouncebuf "descriptor". */
1076 sume_fill_bb_desc(adapter, send, SUME_RIFFA_LEN(send->len));
1078 /* Update the state before intiating the DMA to avoid races. */
1079 send->state = SUME_RIFFA_CHAN_STATE_READY;
1082 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA,
1083 RIFFA_RX_SG_ADDR_LO_REG_OFF),
1084 SUME_RIFFA_LO_ADDR(send->buf_hw_addr));
1085 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA,
1086 RIFFA_RX_SG_ADDR_HI_REG_OFF),
1087 SUME_RIFFA_HI_ADDR(send->buf_hw_addr));
1088 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA,
1089 RIFFA_RX_SG_LEN_REG_OFF), 4 * send->num_sg);
1091 bus_dmamap_sync(send->ch_tag, send->ch_map,
1092 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1094 nf_priv->stats.tx_packets++;
1095 nf_priv->stats.tx_bytes += plen;
1097 /* We can free as long as we use the bounce buffer. */
1100 adapter->last_ifc = nf_priv->port;
1102 /* Reset watchdog counter. */
1103 adapter->wd_counter = 0;
1109 sume_if_start(struct ifnet *ifp)
1111 struct nf_priv *nf_priv = ifp->if_softc;
1112 struct sume_adapter *adapter = nf_priv->adapter;
1114 if (!adapter->running || !(ifp->if_flags & IFF_UP))
1118 if (adapter->send[SUME_RIFFA_CHANNEL_DATA]->state ==
1119 SUME_RIFFA_CHAN_STATE_IDLE)
1120 sume_if_start_locked(ifp);
1121 SUME_UNLOCK(adapter);
1125 * We call this function at the end of every TX transaction to check for
1126 * remaining packets in the TX queues for every UP interface.
1129 check_tx_queues(struct sume_adapter *adapter)
1133 KASSERT(mtx_owned(&adapter->lock), ("SUME lock not owned"));
1135 last_ifc = adapter->last_ifc;
1137 /* Check all interfaces */
1138 for (i = last_ifc + 1; i < last_ifc + SUME_NPORTS + 1; i++) {
1139 struct ifnet *ifp = adapter->ifp[i % SUME_NPORTS];
1141 if (!(ifp->if_flags & IFF_UP))
1144 if (!sume_if_start_locked(ifp))
1150 sume_ifp_alloc(struct sume_adapter *adapter, uint32_t port)
1153 struct nf_priv *nf_priv = malloc(sizeof(struct nf_priv), M_SUME,
1156 ifp = if_alloc(IFT_ETHER);
1158 device_printf(adapter->dev, "cannot allocate ifnet\n");
1162 adapter->ifp[port] = ifp;
1163 ifp->if_softc = nf_priv;
1165 nf_priv->adapter = adapter;
1166 nf_priv->unit = alloc_unr(unr);
1167 nf_priv->port = port;
1168 nf_priv->link_up = 0;
1170 if_initname(ifp, SUME_ETH_DEVICE_NAME, nf_priv->unit);
1171 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1173 ifp->if_init = sume_if_init;
1174 ifp->if_start = sume_if_start;
1175 ifp->if_ioctl = sume_if_ioctl;
1177 uint8_t hw_addr[ETHER_ADDR_LEN] = DEFAULT_ETHER_ADDRESS;
1178 hw_addr[ETHER_ADDR_LEN-1] = nf_priv->unit;
1179 ether_ifattach(ifp, hw_addr);
1181 ifmedia_init(&nf_priv->media, IFM_IMASK, sume_media_change,
1183 ifmedia_add(&nf_priv->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
1184 ifmedia_set(&nf_priv->media, IFM_ETHER | IFM_10G_SR);
1186 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1192 callback_dma(void *arg, bus_dma_segment_t *segs, int nseg, int err)
1197 KASSERT(nseg == 1, ("%d segments returned!", nseg));
1199 *(bus_addr_t *) arg = segs[0].ds_addr;
1203 sume_probe_riffa_buffer(const struct sume_adapter *adapter,
1204 struct riffa_chnl_dir ***p, const char *dir)
1206 struct riffa_chnl_dir **rp;
1209 device_t dev = adapter->dev;
1212 *p = malloc(SUME_RIFFA_CHANNELS * sizeof(struct riffa_chnl_dir *),
1213 M_SUME, M_ZERO | M_WAITOK);
1215 device_printf(dev, "malloc(%s) failed.\n", dir);
1220 /* Allocate the chnl_dir structs themselves. */
1221 for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) {
1222 /* One direction. */
1223 rp[ch] = malloc(sizeof(struct riffa_chnl_dir), M_SUME,
1225 if (rp[ch] == NULL) {
1226 device_printf(dev, "malloc(%s[%d]) riffa_chnl_dir "
1227 "failed.\n", dir, ch);
1231 int err = bus_dma_tag_create(bus_get_dma_tag(dev),
1236 adapter->sg_buf_size,
1238 adapter->sg_buf_size,
1245 device_printf(dev, "bus_dma_tag_create(%s[%d]) "
1246 "failed.\n", dir, ch);
1250 err = bus_dmamem_alloc(rp[ch]->ch_tag, (void **)
1251 &rp[ch]->buf_addr, BUS_DMA_WAITOK | BUS_DMA_COHERENT |
1252 BUS_DMA_ZERO, &rp[ch]->ch_map);
1254 device_printf(dev, "bus_dmamem_alloc(%s[%d]) failed.\n",
1259 bzero(rp[ch]->buf_addr, adapter->sg_buf_size);
1261 err = bus_dmamap_load(rp[ch]->ch_tag, rp[ch]->ch_map,
1262 rp[ch]->buf_addr, adapter->sg_buf_size, callback_dma,
1263 &hw_addr, BUS_DMA_NOWAIT);
1265 device_printf(dev, "bus_dmamap_load(%s[%d]) failed.\n",
1269 rp[ch]->buf_hw_addr = hw_addr;
1271 rp[ch]->state = SUME_RIFFA_CHAN_STATE_IDLE;
1273 rp[ch]->rtag = SUME_INIT_RTAG;
1280 sume_probe_riffa_buffers(struct sume_adapter *adapter)
1284 error = sume_probe_riffa_buffer(adapter, &adapter->recv, "recv");
1288 error = sume_probe_riffa_buffer(adapter, &adapter->send, "send");
1294 sume_sysctl_init(struct sume_adapter *adapter)
1296 device_t dev = adapter->dev;
1297 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
1298 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
1299 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
1300 struct sysctl_oid *tmp_tree;
1301 char namebuf[MAX_IFC_NAME_LEN];
1304 tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "sume", CTLFLAG_RW,
1305 0, "SUME top-level tree");
1307 device_printf(dev, "SYSCTL_ADD_NODE failed.\n");
1310 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug", CTLFLAG_RW,
1311 &adapter->sume_debug, 0, "debug int leaf");
1313 /* total RX error stats */
1314 SYSCTL_ADD_U64(ctx, child, OID_AUTO, "rx_epkts",
1315 CTLFLAG_RD, &adapter->packets_err, 0, "rx errors");
1316 SYSCTL_ADD_U64(ctx, child, OID_AUTO, "rx_ebytes",
1317 CTLFLAG_RD, &adapter->bytes_err, 0, "rx error bytes");
1319 for (i = SUME_NPORTS - 1; i >= 0; i--) {
1320 struct ifnet *ifp = adapter->ifp[i];
1324 struct nf_priv *nf_priv = ifp->if_softc;
1326 snprintf(namebuf, MAX_IFC_NAME_LEN, "%s%d",
1327 SUME_ETH_DEVICE_NAME, nf_priv->unit);
1328 tmp_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1329 CTLFLAG_RW, 0, "SUME ifc tree");
1330 if (tmp_tree == NULL) {
1331 device_printf(dev, "SYSCTL_ADD_NODE failed.\n");
1335 /* Packets dropped by down interface. */
1336 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1337 "ifc_down_bytes", CTLFLAG_RD,
1338 &nf_priv->stats.ifc_down_bytes, 0, "ifc_down bytes");
1339 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1340 "ifc_down_packets", CTLFLAG_RD,
1341 &nf_priv->stats.ifc_down_packets, 0, "ifc_down packets");
1344 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1345 "hw_rx_packets", CTLFLAG_RD, &nf_priv->stats.hw_rx_packets,
1346 0, "hw_rx packets");
1349 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1350 "hw_tx_packets", CTLFLAG_RD, &nf_priv->stats.hw_tx_packets,
1351 0, "hw_tx packets");
1354 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1355 "rx_bytes", CTLFLAG_RD, &nf_priv->stats.rx_bytes, 0,
1357 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1358 "rx_dropped", CTLFLAG_RD, &nf_priv->stats.rx_dropped, 0,
1360 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1361 "rx_packets", CTLFLAG_RD, &nf_priv->stats.rx_packets, 0,
1365 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1366 "tx_bytes", CTLFLAG_RD, &nf_priv->stats.tx_bytes, 0,
1368 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1369 "tx_dropped", CTLFLAG_RD, &nf_priv->stats.tx_dropped, 0,
1371 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO,
1372 "tx_packets", CTLFLAG_RD, &nf_priv->stats.tx_packets, 0,
1378 sume_local_timer(void *arg)
1380 struct sume_adapter *adapter = arg;
1382 if (!adapter->running)
1385 taskqueue_enqueue(adapter->tq, &adapter->stat_task);
1388 if (adapter->send[SUME_RIFFA_CHANNEL_DATA]->state !=
1389 SUME_RIFFA_CHAN_STATE_IDLE && ++adapter->wd_counter >= 3) {
1390 /* Resetting interfaces if stuck for 3 seconds. */
1391 device_printf(adapter->dev, "TX stuck, resetting adapter.\n");
1392 read_reg(adapter, RIFFA_INFO_REG_OFF);
1394 adapter->send[SUME_RIFFA_CHANNEL_DATA]->state =
1395 SUME_RIFFA_CHAN_STATE_IDLE;
1396 adapter->wd_counter = 0;
1398 check_tx_queues(adapter);
1400 SUME_UNLOCK(adapter);
1402 callout_reset(&adapter->timer, 1 * hz, sume_local_timer, adapter);
1406 sume_get_stats(void *context, int pending)
1408 struct sume_adapter *adapter = context;
1411 for (i = 0; i < SUME_NPORTS; i++) {
1412 struct ifnet *ifp = adapter->ifp[i];
1414 if (ifp->if_flags & IFF_UP) {
1415 struct nf_priv *nf_priv = ifp->if_softc;
1416 struct sume_ifreq sifr;
1418 sume_update_link_status(ifp);
1420 /* Get RX counter. */
1421 sifr.addr = SUME_STAT_RX_ADDR(nf_priv->port);
1424 if (!get_modreg_value(nf_priv, &sifr))
1425 nf_priv->stats.hw_rx_packets += sifr.val;
1427 /* Get TX counter. */
1428 sifr.addr = SUME_STAT_TX_ADDR(nf_priv->port);
1431 if (!get_modreg_value(nf_priv, &sifr))
1432 nf_priv->stats.hw_tx_packets += sifr.val;
1438 sume_attach(device_t dev)
1440 struct sume_adapter *adapter = device_get_softc(dev);
1444 mtx_init(&adapter->lock, "Global lock", NULL, MTX_DEF);
1446 adapter->running = 0;
1448 /* OK finish up RIFFA. */
1449 error = sume_probe_riffa_pci(adapter);
1453 error = sume_probe_riffa_buffers(adapter);
1457 /* Now do the network interfaces. */
1458 for (i = 0; i < SUME_NPORTS; i++) {
1459 error = sume_ifp_alloc(adapter, i);
1464 /* Register stats and register sysctls. */
1465 sume_sysctl_init(adapter);
1468 read_reg(adapter, RIFFA_INFO_REG_OFF);
1470 /* Ready to go, "enable" IRQ. */
1471 adapter->running = 1;
1473 callout_init(&adapter->timer, 1);
1474 TASK_INIT(&adapter->stat_task, 0, sume_get_stats, adapter);
1476 adapter->tq = taskqueue_create("sume_stats", M_NOWAIT,
1477 taskqueue_thread_enqueue, &adapter->tq);
1478 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s stattaskq",
1479 device_get_nameunit(adapter->dev));
1481 callout_reset(&adapter->timer, 1 * hz, sume_local_timer, adapter);
1492 sume_remove_riffa_buffer(const struct sume_adapter *adapter,
1493 struct riffa_chnl_dir **pp)
1497 for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) {
1501 if (pp[ch]->buf_hw_addr != 0) {
1502 bus_dmamem_free(pp[ch]->ch_tag, pp[ch]->buf_addr,
1504 pp[ch]->buf_hw_addr = 0;
1507 free(pp[ch], M_SUME);
1512 sume_remove_riffa_buffers(struct sume_adapter *adapter)
1514 if (adapter->send != NULL) {
1515 sume_remove_riffa_buffer(adapter, adapter->send);
1516 free(adapter->send, M_SUME);
1517 adapter->send = NULL;
1519 if (adapter->recv != NULL) {
1520 sume_remove_riffa_buffer(adapter, adapter->recv);
1521 free(adapter->recv, M_SUME);
1522 adapter->recv = NULL;
1527 sume_detach(device_t dev)
1529 struct sume_adapter *adapter = device_get_softc(dev);
1531 struct nf_priv *nf_priv;
1533 KASSERT(mtx_initialized(&adapter->lock), ("SUME mutex not "
1535 adapter->running = 0;
1537 /* Drain the stats callout and task queue. */
1538 callout_drain(&adapter->timer);
1541 taskqueue_drain(adapter->tq, &adapter->stat_task);
1542 taskqueue_free(adapter->tq);
1545 for (i = 0; i < SUME_NPORTS; i++) {
1546 struct ifnet *ifp = adapter->ifp[i];
1550 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1551 nf_priv = ifp->if_softc;
1553 if (ifp->if_flags & IFF_UP)
1555 ifmedia_removeall(&nf_priv->media);
1556 free_unr(unr, nf_priv->unit);
1558 ifp->if_flags &= ~IFF_UP;
1559 ether_ifdetach(ifp);
1562 free(nf_priv, M_SUME);
1565 sume_remove_riffa_buffers(adapter);
1567 if (adapter->irq.tag)
1568 bus_teardown_intr(dev, adapter->irq.res, adapter->irq.tag);
1569 if (adapter->irq.res)
1570 bus_release_resource(dev, SYS_RES_IRQ, adapter->irq.rid,
1573 pci_release_msi(dev);
1575 if (adapter->bar0_addr)
1576 bus_release_resource(dev, SYS_RES_MEMORY, adapter->rid,
1577 adapter->bar0_addr);
1579 mtx_destroy(&adapter->lock);
1585 mod_event(module_t mod, int cmd, void *arg)
1589 unr = new_unrhdr(0, INT_MAX, NULL);
1599 static devclass_t sume_devclass;
1601 DRIVER_MODULE(sume, pci, sume_driver, sume_devclass, mod_event, 0);
1602 MODULE_VERSION(sume, 1);