1 /******************************************************************************
3 Copyright (c) 2006-2009, Myricom Inc.
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Neither the name of the Myricom Inc, nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
28 ***************************************************************************/
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/linker.h>
36 #include <sys/firmware.h>
37 #include <sys/endian.h>
38 #include <sys/sockio.h>
40 #include <sys/malloc.h>
42 #include <sys/kernel.h>
44 #include <sys/module.h>
45 #include <sys/socket.h>
46 #include <sys/sysctl.h>
48 #include <sys/taskqueue.h>
50 /* count xmits ourselves, rather than via drbr */
53 #include <net/if_arp.h>
54 #include <net/ethernet.h>
55 #include <net/if_dl.h>
56 #include <net/if_media.h>
60 #include <net/if_types.h>
61 #include <net/if_vlan_var.h>
64 #include <netinet/in_systm.h>
65 #include <netinet/in.h>
66 #include <netinet/ip.h>
67 #include <netinet/tcp.h>
69 #include <machine/bus.h>
70 #include <machine/in_cksum.h>
71 #include <machine/resource.h>
76 #include <dev/pci/pcireg.h>
77 #include <dev/pci/pcivar.h>
78 #include <dev/pci/pci_private.h> /* XXX for pci_cfg_restore */
80 #include <vm/vm.h> /* for pmap_mapdev() */
83 #if defined(__i386) || defined(__amd64)
84 #include <machine/specialreg.h>
87 #include <dev/mxge/mxge_mcp.h>
88 #include <dev/mxge/mcp_gen_header.h>
89 /*#define MXGE_FAKE_IFP*/
90 #include <dev/mxge/if_mxge_var.h>
92 #include <sys/buf_ring.h>
98 static int mxge_nvidia_ecrc_enable = 1;
99 static int mxge_force_firmware = 0;
100 static int mxge_intr_coal_delay = 30;
101 static int mxge_deassert_wait = 1;
102 static int mxge_flow_control = 1;
103 static int mxge_verbose = 0;
104 static int mxge_lro_cnt = 8;
105 static int mxge_ticks;
106 static int mxge_max_slices = 1;
107 static int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
108 static int mxge_always_promisc = 0;
109 static int mxge_initial_mtu = ETHERMTU_JUMBO;
110 static int mxge_throttle = 0;
111 static char *mxge_fw_unaligned = "mxge_ethp_z8e";
112 static char *mxge_fw_aligned = "mxge_eth_z8e";
113 static char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e";
114 static char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e";
116 static int mxge_probe(device_t dev);
117 static int mxge_attach(device_t dev);
118 static int mxge_detach(device_t dev);
119 static int mxge_shutdown(device_t dev);
120 static void mxge_intr(void *arg);
122 static device_method_t mxge_methods[] =
124 /* Device interface */
125 DEVMETHOD(device_probe, mxge_probe),
126 DEVMETHOD(device_attach, mxge_attach),
127 DEVMETHOD(device_detach, mxge_detach),
128 DEVMETHOD(device_shutdown, mxge_shutdown),
132 static driver_t mxge_driver =
136 sizeof(mxge_softc_t),
139 static devclass_t mxge_devclass;
141 /* Declare ourselves to be a child of the PCI bus.*/
142 DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0);
143 MODULE_DEPEND(mxge, firmware, 1, 1, 1);
144 MODULE_DEPEND(mxge, zlib, 1, 1, 1);
146 static int mxge_load_firmware(mxge_softc_t *sc, int adopt);
147 static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data);
148 static int mxge_close(mxge_softc_t *sc, int down);
149 static int mxge_open(mxge_softc_t *sc);
150 static void mxge_tick(void *arg);
153 mxge_probe(device_t dev)
158 if ((pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM) &&
159 ((pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E) ||
160 (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9))) {
161 rev = pci_get_revid(dev);
163 case MXGE_PCI_REV_Z8E:
164 device_set_desc(dev, "Myri10G-PCIE-8A");
166 case MXGE_PCI_REV_Z8ES:
167 device_set_desc(dev, "Myri10G-PCIE-8B");
170 device_set_desc(dev, "Myri10G-PCIE-8??");
171 device_printf(dev, "Unrecognized rev %d NIC\n",
181 mxge_enable_wc(mxge_softc_t *sc)
183 #if defined(__i386) || defined(__amd64)
188 len = rman_get_size(sc->mem_res);
189 err = pmap_change_attr((vm_offset_t) sc->sram,
190 len, PAT_WRITE_COMBINING);
192 device_printf(sc->dev, "pmap_change_attr failed, %d\n",
200 /* callback to get our DMA address */
202 mxge_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs,
206 *(bus_addr_t *) arg = segs->ds_addr;
211 mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes,
212 bus_size_t alignment)
215 device_t dev = sc->dev;
216 bus_size_t boundary, maxsegsize;
218 if (bytes > 4096 && alignment == 4096) {
226 /* allocate DMAable memory tags */
227 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
228 alignment, /* alignment */
229 boundary, /* boundary */
230 BUS_SPACE_MAXADDR, /* low */
231 BUS_SPACE_MAXADDR, /* high */
232 NULL, NULL, /* filter */
235 maxsegsize, /* maxsegsize */
236 BUS_DMA_COHERENT, /* flags */
237 NULL, NULL, /* lock */
238 &dma->dmat); /* tag */
240 device_printf(dev, "couldn't alloc tag (err = %d)\n", err);
244 /* allocate DMAable memory & map */
245 err = bus_dmamem_alloc(dma->dmat, &dma->addr,
246 (BUS_DMA_WAITOK | BUS_DMA_COHERENT
247 | BUS_DMA_ZERO), &dma->map);
249 device_printf(dev, "couldn't alloc mem (err = %d)\n", err);
250 goto abort_with_dmat;
253 /* load the memory */
254 err = bus_dmamap_load(dma->dmat, dma->map, dma->addr, bytes,
255 mxge_dmamap_callback,
256 (void *)&dma->bus_addr, 0);
258 device_printf(dev, "couldn't load map (err = %d)\n", err);
264 bus_dmamem_free(dma->dmat, dma->addr, dma->map);
266 (void)bus_dma_tag_destroy(dma->dmat);
272 mxge_dma_free(mxge_dma_t *dma)
274 bus_dmamap_unload(dma->dmat, dma->map);
275 bus_dmamem_free(dma->dmat, dma->addr, dma->map);
276 (void)bus_dma_tag_destroy(dma->dmat);
280 * The eeprom strings on the lanaiX have the format
287 mxge_parse_strings(mxge_softc_t *sc)
289 #define MXGE_NEXT_STRING(p) while(ptr < limit && *ptr++)
294 ptr = sc->eeprom_strings;
295 limit = sc->eeprom_strings + MXGE_EEPROM_STRINGS_SIZE;
297 while (ptr < limit && *ptr != '\0') {
298 if (memcmp(ptr, "MAC=", 4) == 0) {
300 sc->mac_addr_string = ptr;
301 for (i = 0; i < 6; i++) {
303 if ((ptr + 2) > limit)
305 sc->mac_addr[i] = strtoul(ptr, NULL, 16);
308 } else if (memcmp(ptr, "PC=", 3) == 0) {
310 strncpy(sc->product_code_string, ptr,
311 sizeof (sc->product_code_string) - 1);
312 } else if (memcmp(ptr, "SN=", 3) == 0) {
314 strncpy(sc->serial_number_string, ptr,
315 sizeof (sc->serial_number_string) - 1);
317 MXGE_NEXT_STRING(ptr);
324 device_printf(sc->dev, "failed to parse eeprom_strings\n");
329 #if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__
331 mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
334 unsigned long base, off;
336 device_t pdev, mcp55;
337 uint16_t vendor_id, device_id, word;
338 uintptr_t bus, slot, func, ivend, idev;
342 if (!mxge_nvidia_ecrc_enable)
345 pdev = device_get_parent(device_get_parent(sc->dev));
347 device_printf(sc->dev, "could not find parent?\n");
350 vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2);
351 device_id = pci_read_config(pdev, PCIR_DEVICE, 2);
353 if (vendor_id != 0x10de)
358 if (device_id == 0x005d) {
359 /* ck804, base address is magic */
361 } else if (device_id >= 0x0374 && device_id <= 0x378) {
362 /* mcp55, base address stored in chipset */
363 mcp55 = pci_find_bsf(0, 0, 0);
365 0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) &&
366 0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) {
367 word = pci_read_config(mcp55, 0x90, 2);
368 base = ((unsigned long)word & 0x7ffeU) << 25;
375 Test below is commented because it is believed that doing
376 config read/write beyond 0xff will access the config space
377 for the next larger function. Uncomment this and remove
378 the hacky pmap_mapdev() way of accessing config space when
379 FreeBSD grows support for extended pcie config space access
382 /* See if we can, by some miracle, access the extended
384 val = pci_read_config(pdev, 0x178, 4);
385 if (val != 0xffffffff) {
387 pci_write_config(pdev, 0x178, val, 4);
391 /* Rather than using normal pci config space writes, we must
392 * map the Nvidia config space ourselves. This is because on
393 * opteron/nvidia class machine the 0xe000000 mapping is
394 * handled by the nvidia chipset, that means the internal PCI
395 * device (the on-chip northbridge), or the amd-8131 bridge
396 * and things behind them are not visible by this method.
399 BUS_READ_IVAR(device_get_parent(pdev), pdev,
401 BUS_READ_IVAR(device_get_parent(pdev), pdev,
402 PCI_IVAR_SLOT, &slot);
403 BUS_READ_IVAR(device_get_parent(pdev), pdev,
404 PCI_IVAR_FUNCTION, &func);
405 BUS_READ_IVAR(device_get_parent(pdev), pdev,
406 PCI_IVAR_VENDOR, &ivend);
407 BUS_READ_IVAR(device_get_parent(pdev), pdev,
408 PCI_IVAR_DEVICE, &idev);
411 + 0x00100000UL * (unsigned long)bus
412 + 0x00001000UL * (unsigned long)(func
415 /* map it into the kernel */
416 va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE);
420 device_printf(sc->dev, "pmap_kenter_temporary didn't\n");
423 /* get a pointer to the config space mapped into the kernel */
424 cfgptr = va + (off & PAGE_MASK);
426 /* make sure that we can really access it */
427 vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR);
428 device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE);
429 if (! (vendor_id == ivend && device_id == idev)) {
430 device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n",
431 vendor_id, device_id);
432 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
436 ptr32 = (uint32_t*)(cfgptr + 0x178);
439 if (val == 0xffffffff) {
440 device_printf(sc->dev, "extended mapping failed\n");
441 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
445 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
447 device_printf(sc->dev,
448 "Enabled ECRC on upstream Nvidia bridge "
450 (int)bus, (int)slot, (int)func);
455 mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
457 device_printf(sc->dev,
458 "Nforce 4 chipset on non-x86/amd64!?!?!\n");
465 mxge_dma_test(mxge_softc_t *sc, int test_type)
468 bus_addr_t dmatest_bus = sc->dmabench_dma.bus_addr;
474 /* Run a small DMA test.
475 * The magic multipliers to the length tell the firmware
476 * to do DMA read, write, or read+write tests. The
477 * results are returned in cmd.data0. The upper 16
478 * bits of the return is the number of transfers completed.
479 * The lower 16 bits is the time in 0.5us ticks that the
480 * transfers took to complete.
483 len = sc->tx_boundary;
485 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
486 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
487 cmd.data2 = len * 0x10000;
488 status = mxge_send_cmd(sc, test_type, &cmd);
493 sc->read_dma = ((cmd.data0>>16) * len * 2) /
494 (cmd.data0 & 0xffff);
495 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
496 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
497 cmd.data2 = len * 0x1;
498 status = mxge_send_cmd(sc, test_type, &cmd);
503 sc->write_dma = ((cmd.data0>>16) * len * 2) /
504 (cmd.data0 & 0xffff);
506 cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
507 cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
508 cmd.data2 = len * 0x10001;
509 status = mxge_send_cmd(sc, test_type, &cmd);
514 sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) /
515 (cmd.data0 & 0xffff);
518 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST)
519 device_printf(sc->dev, "DMA %s benchmark failed: %d\n",
526 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput
527 * when the PCI-E Completion packets are aligned on an 8-byte
528 * boundary. Some PCI-E chip sets always align Completion packets; on
529 * the ones that do not, the alignment can be enforced by enabling
530 * ECRC generation (if supported).
532 * When PCI-E Completion packets are not aligned, it is actually more
533 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
535 * If the driver can neither enable ECRC nor verify that it has
536 * already been enabled, then it must use a firmware image which works
537 * around unaligned completion packets (ethp_z8e.dat), and it should
538 * also ensure that it never gives the device a Read-DMA which is
539 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is
540 * enabled, then the driver should use the aligned (eth_z8e.dat)
541 * firmware image, and set tx_boundary to 4KB.
545 mxge_firmware_probe(mxge_softc_t *sc)
547 device_t dev = sc->dev;
551 sc->tx_boundary = 4096;
553 * Verify the max read request size was set to 4KB
554 * before trying the test with 4KB.
556 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
557 pectl = pci_read_config(dev, reg + 0x8, 2);
558 if ((pectl & (5 << 12)) != (5 << 12)) {
559 device_printf(dev, "Max Read Req. size != 4k (0x%x\n",
561 sc->tx_boundary = 2048;
566 * load the optimized firmware (which assumes aligned PCIe
567 * completions) in order to see if it works on this host.
569 sc->fw_name = mxge_fw_aligned;
570 status = mxge_load_firmware(sc, 1);
576 * Enable ECRC if possible
578 mxge_enable_nvidia_ecrc(sc);
581 * Run a DMA test which watches for unaligned completions and
582 * aborts on the first one seen.
585 status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST);
587 return 0; /* keep the aligned firmware */
590 device_printf(dev, "DMA test failed: %d\n", status);
591 if (status == ENOSYS)
592 device_printf(dev, "Falling back to ethp! "
593 "Please install up to date fw\n");
598 mxge_select_firmware(mxge_softc_t *sc)
601 int force_firmware = mxge_force_firmware;
604 force_firmware = sc->throttle;
606 if (force_firmware != 0) {
607 if (force_firmware == 1)
612 device_printf(sc->dev,
613 "Assuming %s completions (forced)\n",
614 aligned ? "aligned" : "unaligned");
618 /* if the PCIe link width is 4 or less, we can use the aligned
619 firmware and skip any checks */
620 if (sc->link_width != 0 && sc->link_width <= 4) {
621 device_printf(sc->dev,
622 "PCIe x%d Link, expect reduced performance\n",
628 if (0 == mxge_firmware_probe(sc))
633 sc->fw_name = mxge_fw_aligned;
634 sc->tx_boundary = 4096;
636 sc->fw_name = mxge_fw_unaligned;
637 sc->tx_boundary = 2048;
639 return (mxge_load_firmware(sc, 0));
649 mxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr)
653 if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) {
654 device_printf(sc->dev, "Bad firmware type: 0x%x\n",
655 be32toh(hdr->mcp_type));
659 /* save firmware version for sysctl */
660 strncpy(sc->fw_version, hdr->version, sizeof (sc->fw_version));
662 device_printf(sc->dev, "firmware id: %s\n", hdr->version);
664 sscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major,
665 &sc->fw_ver_minor, &sc->fw_ver_tiny);
667 if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR
668 && sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) {
669 device_printf(sc->dev, "Found firmware version %s\n",
671 device_printf(sc->dev, "Driver needs %d.%d\n",
672 MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR);
680 z_alloc(void *nil, u_int items, u_int size)
684 ptr = malloc(items * size, M_TEMP, M_NOWAIT);
689 z_free(void *nil, void *ptr)
696 mxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit)
699 char *inflate_buffer;
700 const struct firmware *fw;
701 const mcp_gen_header_t *hdr;
708 fw = firmware_get(sc->fw_name);
710 device_printf(sc->dev, "Could not find firmware image %s\n",
717 /* setup zlib and decompress f/w */
718 bzero(&zs, sizeof (zs));
721 status = inflateInit(&zs);
722 if (status != Z_OK) {
727 /* the uncompressed size is stored as the firmware version,
728 which would otherwise go unused */
729 fw_len = (size_t) fw->version;
730 inflate_buffer = malloc(fw_len, M_TEMP, M_NOWAIT);
731 if (inflate_buffer == NULL)
733 zs.avail_in = fw->datasize;
734 zs.next_in = __DECONST(char *, fw->data);
735 zs.avail_out = fw_len;
736 zs.next_out = inflate_buffer;
737 status = inflate(&zs, Z_FINISH);
738 if (status != Z_STREAM_END) {
739 device_printf(sc->dev, "zlib %d\n", status);
741 goto abort_with_buffer;
745 hdr_offset = htobe32(*(const uint32_t *)
746 (inflate_buffer + MCP_HEADER_PTR_OFFSET));
747 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) {
748 device_printf(sc->dev, "Bad firmware file");
750 goto abort_with_buffer;
752 hdr = (const void*)(inflate_buffer + hdr_offset);
754 status = mxge_validate_firmware(sc, hdr);
756 goto abort_with_buffer;
758 /* Copy the inflated firmware to NIC SRAM. */
759 for (i = 0; i < fw_len; i += 256) {
760 mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i,
762 min(256U, (unsigned)(fw_len - i)));
771 free(inflate_buffer, M_TEMP);
775 firmware_put(fw, FIRMWARE_UNLOAD);
780 * Enable or disable periodic RDMAs from the host to make certain
781 * chipsets resend dropped PCIe messages
785 mxge_dummy_rdma(mxge_softc_t *sc, int enable)
788 volatile uint32_t *confirm;
789 volatile char *submit;
790 uint32_t *buf, dma_low, dma_high;
793 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
795 /* clear confirmation addr */
796 confirm = (volatile uint32_t *)sc->cmd;
800 /* send an rdma command to the PCIe engine, and wait for the
801 response in the confirmation address. The firmware should
802 write a -1 there to indicate it is alive and well
805 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
806 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
807 buf[0] = htobe32(dma_high); /* confirm addr MSW */
808 buf[1] = htobe32(dma_low); /* confirm addr LSW */
809 buf[2] = htobe32(0xffffffff); /* confirm data */
810 dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr);
811 dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr);
812 buf[3] = htobe32(dma_high); /* dummy addr MSW */
813 buf[4] = htobe32(dma_low); /* dummy addr LSW */
814 buf[5] = htobe32(enable); /* enable? */
817 submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA);
819 mxge_pio_copy(submit, buf, 64);
824 while (*confirm != 0xffffffff && i < 20) {
828 if (*confirm != 0xffffffff) {
829 device_printf(sc->dev, "dummy rdma %s failed (%p = 0x%x)",
830 (enable ? "enable" : "disable"), confirm,
837 mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data)
840 char buf_bytes[sizeof(*buf) + 8];
841 volatile mcp_cmd_response_t *response = sc->cmd;
842 volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD;
843 uint32_t dma_low, dma_high;
844 int err, sleep_total = 0;
846 /* ensure buf is aligned to 8 bytes */
847 buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
849 buf->data0 = htobe32(data->data0);
850 buf->data1 = htobe32(data->data1);
851 buf->data2 = htobe32(data->data2);
852 buf->cmd = htobe32(cmd);
853 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
854 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
856 buf->response_addr.low = htobe32(dma_low);
857 buf->response_addr.high = htobe32(dma_high);
858 mtx_lock(&sc->cmd_mtx);
859 response->result = 0xffffffff;
861 mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf));
863 /* wait up to 20ms */
865 for (sleep_total = 0; sleep_total < 20; sleep_total++) {
866 bus_dmamap_sync(sc->cmd_dma.dmat,
867 sc->cmd_dma.map, BUS_DMASYNC_POSTREAD);
869 switch (be32toh(response->result)) {
871 data->data0 = be32toh(response->data);
877 case MXGEFW_CMD_UNKNOWN:
880 case MXGEFW_CMD_ERROR_UNALIGNED:
883 case MXGEFW_CMD_ERROR_BUSY:
886 case MXGEFW_CMD_ERROR_I2C_ABSENT:
890 device_printf(sc->dev,
892 "failed, result = %d\n",
893 cmd, be32toh(response->result));
901 device_printf(sc->dev, "mxge: command %d timed out"
903 cmd, be32toh(response->result));
904 mtx_unlock(&sc->cmd_mtx);
909 mxge_adopt_running_firmware(mxge_softc_t *sc)
911 struct mcp_gen_header *hdr;
912 const size_t bytes = sizeof (struct mcp_gen_header);
916 /* find running firmware header */
917 hdr_offset = htobe32(*(volatile uint32_t *)
918 (sc->sram + MCP_HEADER_PTR_OFFSET));
920 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) {
921 device_printf(sc->dev,
922 "Running firmware has bad header offset (%d)\n",
927 /* copy header of running firmware from SRAM to host memory to
928 * validate firmware */
929 hdr = malloc(bytes, M_DEVBUF, M_NOWAIT);
931 device_printf(sc->dev, "could not malloc firmware hdr\n");
934 bus_space_read_region_1(rman_get_bustag(sc->mem_res),
935 rman_get_bushandle(sc->mem_res),
936 hdr_offset, (char *)hdr, bytes);
937 status = mxge_validate_firmware(sc, hdr);
941 * check to see if adopted firmware has bug where adopting
942 * it will cause broadcasts to be filtered unless the NIC
943 * is kept in ALLMULTI mode
945 if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 &&
946 sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) {
947 sc->adopted_rx_filter_bug = 1;
948 device_printf(sc->dev, "Adopting fw %d.%d.%d: "
949 "working around rx filter bug\n",
950 sc->fw_ver_major, sc->fw_ver_minor,
959 mxge_load_firmware(mxge_softc_t *sc, int adopt)
961 volatile uint32_t *confirm;
962 volatile char *submit;
964 uint32_t *buf, size, dma_low, dma_high;
967 buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
969 size = sc->sram_size;
970 status = mxge_load_firmware_helper(sc, &size);
974 /* Try to use the currently running firmware, if
976 status = mxge_adopt_running_firmware(sc);
978 device_printf(sc->dev,
979 "failed to adopt running firmware\n");
982 device_printf(sc->dev,
983 "Successfully adopted running firmware\n");
984 if (sc->tx_boundary == 4096) {
985 device_printf(sc->dev,
986 "Using firmware currently running on NIC"
988 device_printf(sc->dev,
989 "performance consider loading optimized "
992 sc->fw_name = mxge_fw_unaligned;
993 sc->tx_boundary = 2048;
996 /* clear confirmation addr */
997 confirm = (volatile uint32_t *)sc->cmd;
1000 /* send a reload command to the bootstrap MCP, and wait for the
1001 response in the confirmation address. The firmware should
1002 write a -1 there to indicate it is alive and well
1005 dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
1006 dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
1008 buf[0] = htobe32(dma_high); /* confirm addr MSW */
1009 buf[1] = htobe32(dma_low); /* confirm addr LSW */
1010 buf[2] = htobe32(0xffffffff); /* confirm data */
1012 /* FIX: All newest firmware should un-protect the bottom of
1013 the sram before handoff. However, the very first interfaces
1014 do not. Therefore the handoff copy must skip the first 8 bytes
1016 /* where the code starts*/
1017 buf[3] = htobe32(MXGE_FW_OFFSET + 8);
1018 buf[4] = htobe32(size - 8); /* length of code */
1019 buf[5] = htobe32(8); /* where to copy to */
1020 buf[6] = htobe32(0); /* where to jump to */
1022 submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF);
1023 mxge_pio_copy(submit, buf, 64);
1028 while (*confirm != 0xffffffff && i < 20) {
1031 bus_dmamap_sync(sc->cmd_dma.dmat,
1032 sc->cmd_dma.map, BUS_DMASYNC_POSTREAD);
1034 if (*confirm != 0xffffffff) {
1035 device_printf(sc->dev,"handoff failed (%p = 0x%x)",
1044 mxge_update_mac_address(mxge_softc_t *sc)
1047 uint8_t *addr = sc->mac_addr;
1051 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16)
1052 | (addr[2] << 8) | addr[3]);
1054 cmd.data1 = ((addr[4] << 8) | (addr[5]));
1056 status = mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd);
1061 mxge_change_pause(mxge_softc_t *sc, int pause)
1067 status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL,
1070 status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL,
1074 device_printf(sc->dev, "Failed to set flow control mode\n");
1082 mxge_change_promisc(mxge_softc_t *sc, int promisc)
1087 if (mxge_always_promisc)
1091 status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC,
1094 status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC,
1098 device_printf(sc->dev, "Failed to set promisc mode\n");
1103 mxge_set_multicast_list(mxge_softc_t *sc)
1106 struct ifmultiaddr *ifma;
1107 struct ifnet *ifp = sc->ifp;
1110 /* This firmware is known to not support multicast */
1111 if (!sc->fw_multicast_support)
1114 /* Disable multicast filtering while we play with the lists*/
1115 err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd);
1117 device_printf(sc->dev, "Failed MXGEFW_ENABLE_ALLMULTI,"
1118 " error status: %d\n", err);
1122 if (sc->adopted_rx_filter_bug)
1125 if (ifp->if_flags & IFF_ALLMULTI)
1126 /* request to disable multicast filtering, so quit here */
1129 /* Flush all the filters */
1131 err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd);
1133 device_printf(sc->dev,
1134 "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS"
1135 ", error status: %d\n", err);
1139 /* Walk the multicast list, and add each address */
1141 if_maddr_rlock(ifp);
1142 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1143 if (ifma->ifma_addr->sa_family != AF_LINK)
1145 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1147 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4,
1149 cmd.data0 = htonl(cmd.data0);
1150 cmd.data1 = htonl(cmd.data1);
1151 err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd);
1153 device_printf(sc->dev, "Failed "
1154 "MXGEFW_JOIN_MULTICAST_GROUP, error status:"
1156 /* abort, leaving multicast filtering off */
1157 if_maddr_runlock(ifp);
1161 if_maddr_runlock(ifp);
1162 /* Enable multicast filtering */
1163 err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd);
1165 device_printf(sc->dev, "Failed MXGEFW_DISABLE_ALLMULTI"
1166 ", error status: %d\n", err);
1171 mxge_max_mtu(mxge_softc_t *sc)
1176 if (MJUMPAGESIZE - MXGEFW_PAD > MXGEFW_MAX_MTU)
1177 return MXGEFW_MAX_MTU - MXGEFW_PAD;
1179 /* try to set nbufs to see if it we can
1180 use virtually contiguous jumbos */
1182 status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
1185 return MXGEFW_MAX_MTU - MXGEFW_PAD;
1187 /* otherwise, we're limited to MJUMPAGESIZE */
1188 return MJUMPAGESIZE - MXGEFW_PAD;
1192 mxge_reset(mxge_softc_t *sc, int interrupts_setup)
1194 struct mxge_slice_state *ss;
1195 mxge_rx_done_t *rx_done;
1196 volatile uint32_t *irq_claim;
1200 /* try to send a reset command to the card to see if it
1202 memset(&cmd, 0, sizeof (cmd));
1203 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
1205 device_printf(sc->dev, "failed reset\n");
1209 mxge_dummy_rdma(sc, 1);
1212 /* set the intrq size */
1213 cmd.data0 = sc->rx_ring_size;
1214 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
1217 * Even though we already know how many slices are supported
1218 * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES
1219 * has magic side effects, and must be called after a reset.
1220 * It must be called prior to calling any RSS related cmds,
1221 * including assigning an interrupt queue for anything but
1222 * slice 0. It must also be called *after*
1223 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
1224 * the firmware to compute offsets.
1227 if (sc->num_slices > 1) {
1228 /* ask the maximum number of slices it supports */
1229 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
1232 device_printf(sc->dev,
1233 "failed to get number of slices\n");
1237 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
1238 * to setting up the interrupt queue DMA
1240 cmd.data0 = sc->num_slices;
1241 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
1242 #ifdef IFNET_BUF_RING
1243 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
1245 status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES,
1248 device_printf(sc->dev,
1249 "failed to set number of slices\n");
1255 if (interrupts_setup) {
1256 /* Now exchange information about interrupts */
1257 for (slice = 0; slice < sc->num_slices; slice++) {
1258 rx_done = &sc->ss[slice].rx_done;
1259 memset(rx_done->entry, 0, sc->rx_ring_size);
1260 cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr);
1261 cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr);
1263 status |= mxge_send_cmd(sc,
1264 MXGEFW_CMD_SET_INTRQ_DMA,
1269 status |= mxge_send_cmd(sc,
1270 MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd);
1273 sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0);
1275 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
1276 irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0);
1279 status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET,
1281 sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0);
1283 device_printf(sc->dev, "failed set interrupt parameters\n");
1288 *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay);
1291 /* run a DMA benchmark */
1292 (void) mxge_dma_test(sc, MXGEFW_DMA_TEST);
1294 for (slice = 0; slice < sc->num_slices; slice++) {
1295 ss = &sc->ss[slice];
1297 ss->irq_claim = irq_claim + (2 * slice);
1298 /* reset mcp/driver shared state back to 0 */
1299 ss->rx_done.idx = 0;
1300 ss->rx_done.cnt = 0;
1303 ss->tx.pkt_done = 0;
1304 ss->tx.queue_active = 0;
1305 ss->tx.activate = 0;
1306 ss->tx.deactivate = 0;
1311 ss->rx_small.cnt = 0;
1312 ss->lro_bad_csum = 0;
1314 ss->lro_flushed = 0;
1315 if (ss->fw_stats != NULL) {
1316 bzero(ss->fw_stats, sizeof *ss->fw_stats);
1319 sc->rdma_tags_available = 15;
1320 status = mxge_update_mac_address(sc);
1321 mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC);
1322 mxge_change_pause(sc, sc->pause);
1323 mxge_set_multicast_list(sc);
1325 cmd.data0 = sc->throttle;
1326 if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR,
1328 device_printf(sc->dev,
1329 "can't enable throttle\n");
1336 mxge_change_throttle(SYSCTL_HANDLER_ARGS)
1341 unsigned int throttle;
1344 throttle = sc->throttle;
1345 err = sysctl_handle_int(oidp, &throttle, arg2, req);
1350 if (throttle == sc->throttle)
1353 if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE)
1356 mtx_lock(&sc->driver_mtx);
1357 cmd.data0 = throttle;
1358 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd);
1360 sc->throttle = throttle;
1361 mtx_unlock(&sc->driver_mtx);
1366 mxge_change_intr_coal(SYSCTL_HANDLER_ARGS)
1369 unsigned int intr_coal_delay;
1373 intr_coal_delay = sc->intr_coal_delay;
1374 err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req);
1378 if (intr_coal_delay == sc->intr_coal_delay)
1381 if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000)
1384 mtx_lock(&sc->driver_mtx);
1385 *sc->intr_coal_delay_ptr = htobe32(intr_coal_delay);
1386 sc->intr_coal_delay = intr_coal_delay;
1388 mtx_unlock(&sc->driver_mtx);
1393 mxge_change_flow_control(SYSCTL_HANDLER_ARGS)
1396 unsigned int enabled;
1400 enabled = sc->pause;
1401 err = sysctl_handle_int(oidp, &enabled, arg2, req);
1405 if (enabled == sc->pause)
1408 mtx_lock(&sc->driver_mtx);
1409 err = mxge_change_pause(sc, enabled);
1410 mtx_unlock(&sc->driver_mtx);
1415 mxge_change_lro_locked(mxge_softc_t *sc, int lro_cnt)
1422 ifp->if_capenable &= ~IFCAP_LRO;
1424 ifp->if_capenable |= IFCAP_LRO;
1425 sc->lro_cnt = lro_cnt;
1426 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1428 err = mxge_open(sc);
1434 mxge_change_lro(SYSCTL_HANDLER_ARGS)
1437 unsigned int lro_cnt;
1441 lro_cnt = sc->lro_cnt;
1442 err = sysctl_handle_int(oidp, &lro_cnt, arg2, req);
1446 if (lro_cnt == sc->lro_cnt)
1452 mtx_lock(&sc->driver_mtx);
1453 err = mxge_change_lro_locked(sc, lro_cnt);
1454 mtx_unlock(&sc->driver_mtx);
1459 mxge_handle_be32(SYSCTL_HANDLER_ARGS)
1465 arg2 = be32toh(*(int *)arg1);
1467 err = sysctl_handle_int(oidp, arg1, arg2, req);
1473 mxge_rem_sysctls(mxge_softc_t *sc)
1475 struct mxge_slice_state *ss;
1478 if (sc->slice_sysctl_tree == NULL)
1481 for (slice = 0; slice < sc->num_slices; slice++) {
1482 ss = &sc->ss[slice];
1483 if (ss == NULL || ss->sysctl_tree == NULL)
1485 sysctl_ctx_free(&ss->sysctl_ctx);
1486 ss->sysctl_tree = NULL;
1488 sysctl_ctx_free(&sc->slice_sysctl_ctx);
1489 sc->slice_sysctl_tree = NULL;
1493 mxge_add_sysctls(mxge_softc_t *sc)
1495 struct sysctl_ctx_list *ctx;
1496 struct sysctl_oid_list *children;
1498 struct mxge_slice_state *ss;
1502 ctx = device_get_sysctl_ctx(sc->dev);
1503 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
1504 fw = sc->ss[0].fw_stats;
1506 /* random information */
1507 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1509 CTLFLAG_RD, &sc->fw_version,
1510 0, "firmware version");
1511 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1513 CTLFLAG_RD, &sc->serial_number_string,
1514 0, "serial number");
1515 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1517 CTLFLAG_RD, &sc->product_code_string,
1519 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1521 CTLFLAG_RD, &sc->link_width,
1523 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1525 CTLFLAG_RD, &sc->tx_boundary,
1527 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1529 CTLFLAG_RD, &sc->wc,
1530 0, "write combining PIO?");
1531 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1533 CTLFLAG_RD, &sc->read_dma,
1534 0, "DMA Read speed in MB/s");
1535 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1537 CTLFLAG_RD, &sc->write_dma,
1538 0, "DMA Write speed in MB/s");
1539 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1540 "read_write_dma_MBs",
1541 CTLFLAG_RD, &sc->read_write_dma,
1542 0, "DMA concurrent Read/Write speed in MB/s");
1543 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1545 CTLFLAG_RD, &sc->watchdog_resets,
1546 0, "Number of times NIC was reset");
1549 /* performance related tunables */
1550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1552 CTLTYPE_INT|CTLFLAG_RW, sc,
1553 0, mxge_change_intr_coal,
1554 "I", "interrupt coalescing delay in usecs");
1556 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1558 CTLTYPE_INT|CTLFLAG_RW, sc,
1559 0, mxge_change_throttle,
1560 "I", "transmit throttling");
1562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1563 "flow_control_enabled",
1564 CTLTYPE_INT|CTLFLAG_RW, sc,
1565 0, mxge_change_flow_control,
1566 "I", "interrupt coalescing delay in usecs");
1568 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1570 CTLFLAG_RW, &mxge_deassert_wait,
1571 0, "Wait for IRQ line to go low in ihandler");
1573 /* stats block from firmware is in network byte order.
1575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1577 CTLTYPE_INT|CTLFLAG_RD, &fw->link_up,
1578 0, mxge_handle_be32,
1580 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1581 "rdma_tags_available",
1582 CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available,
1583 0, mxge_handle_be32,
1584 "I", "rdma_tags_available");
1585 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1586 "dropped_bad_crc32",
1587 CTLTYPE_INT|CTLFLAG_RD,
1588 &fw->dropped_bad_crc32,
1589 0, mxge_handle_be32,
1590 "I", "dropped_bad_crc32");
1591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1593 CTLTYPE_INT|CTLFLAG_RD,
1594 &fw->dropped_bad_phy,
1595 0, mxge_handle_be32,
1596 "I", "dropped_bad_phy");
1597 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1598 "dropped_link_error_or_filtered",
1599 CTLTYPE_INT|CTLFLAG_RD,
1600 &fw->dropped_link_error_or_filtered,
1601 0, mxge_handle_be32,
1602 "I", "dropped_link_error_or_filtered");
1603 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1604 "dropped_link_overflow",
1605 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow,
1606 0, mxge_handle_be32,
1607 "I", "dropped_link_overflow");
1608 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1609 "dropped_multicast_filtered",
1610 CTLTYPE_INT|CTLFLAG_RD,
1611 &fw->dropped_multicast_filtered,
1612 0, mxge_handle_be32,
1613 "I", "dropped_multicast_filtered");
1614 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1615 "dropped_no_big_buffer",
1616 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer,
1617 0, mxge_handle_be32,
1618 "I", "dropped_no_big_buffer");
1619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1620 "dropped_no_small_buffer",
1621 CTLTYPE_INT|CTLFLAG_RD,
1622 &fw->dropped_no_small_buffer,
1623 0, mxge_handle_be32,
1624 "I", "dropped_no_small_buffer");
1625 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1627 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun,
1628 0, mxge_handle_be32,
1629 "I", "dropped_overrun");
1630 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1632 CTLTYPE_INT|CTLFLAG_RD,
1634 0, mxge_handle_be32,
1635 "I", "dropped_pause");
1636 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1638 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt,
1639 0, mxge_handle_be32,
1640 "I", "dropped_runt");
1642 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1643 "dropped_unicast_filtered",
1644 CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered,
1645 0, mxge_handle_be32,
1646 "I", "dropped_unicast_filtered");
1648 /* verbose printing? */
1649 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1651 CTLFLAG_RW, &mxge_verbose,
1652 0, "verbose printing");
1655 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1657 CTLTYPE_INT|CTLFLAG_RW, sc,
1659 "I", "number of lro merge queues");
1662 /* add counters exported for debugging from all slices */
1663 sysctl_ctx_init(&sc->slice_sysctl_ctx);
1664 sc->slice_sysctl_tree =
1665 SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO,
1666 "slice", CTLFLAG_RD, 0, "");
1668 for (slice = 0; slice < sc->num_slices; slice++) {
1669 ss = &sc->ss[slice];
1670 sysctl_ctx_init(&ss->sysctl_ctx);
1671 ctx = &ss->sysctl_ctx;
1672 children = SYSCTL_CHILDREN(sc->slice_sysctl_tree);
1673 sprintf(slice_num, "%d", slice);
1675 SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num,
1677 children = SYSCTL_CHILDREN(ss->sysctl_tree);
1678 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1680 CTLFLAG_RD, &ss->rx_small.cnt,
1682 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1684 CTLFLAG_RD, &ss->rx_big.cnt,
1686 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1687 "lro_flushed", CTLFLAG_RD, &ss->lro_flushed,
1688 0, "number of lro merge queues flushed");
1690 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1691 "lro_queued", CTLFLAG_RD, &ss->lro_queued,
1692 0, "number of frames appended to lro merge"
1695 #ifndef IFNET_BUF_RING
1696 /* only transmit from slice 0 for now */
1700 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1702 CTLFLAG_RD, &ss->tx.req,
1705 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1707 CTLFLAG_RD, &ss->tx.done,
1709 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1711 CTLFLAG_RD, &ss->tx.pkt_done,
1713 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1715 CTLFLAG_RD, &ss->tx.stall,
1717 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1719 CTLFLAG_RD, &ss->tx.wake,
1721 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1723 CTLFLAG_RD, &ss->tx.defrag,
1725 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1727 CTLFLAG_RD, &ss->tx.queue_active,
1728 0, "tx_queue_active");
1729 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1731 CTLFLAG_RD, &ss->tx.activate,
1733 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1735 CTLFLAG_RD, &ss->tx.deactivate,
1736 0, "tx_deactivate");
1740 /* copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1741 backwards one at a time and handle ring wraps */
1744 mxge_submit_req_backwards(mxge_tx_ring_t *tx,
1745 mcp_kreq_ether_send_t *src, int cnt)
1747 int idx, starting_slot;
1748 starting_slot = tx->req;
1751 idx = (starting_slot + cnt) & tx->mask;
1752 mxge_pio_copy(&tx->lanai[idx],
1753 &src[cnt], sizeof(*src));
1759 * copy an array of mcp_kreq_ether_send_t's to the mcp. Copy
1760 * at most 32 bytes at a time, so as to avoid involving the software
1761 * pio handler in the nic. We re-write the first segment's flags
1762 * to mark them valid only after writing the entire chain
1766 mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src,
1771 volatile uint32_t *dst_ints;
1772 mcp_kreq_ether_send_t *srcp;
1773 volatile mcp_kreq_ether_send_t *dstp, *dst;
1776 idx = tx->req & tx->mask;
1778 last_flags = src->flags;
1781 dst = dstp = &tx->lanai[idx];
1784 if ((idx + cnt) < tx->mask) {
1785 for (i = 0; i < (cnt - 1); i += 2) {
1786 mxge_pio_copy(dstp, srcp, 2 * sizeof(*src));
1787 wmb(); /* force write every 32 bytes */
1792 /* submit all but the first request, and ensure
1793 that it is submitted below */
1794 mxge_submit_req_backwards(tx, src, cnt);
1798 /* submit the first request */
1799 mxge_pio_copy(dstp, srcp, sizeof(*src));
1800 wmb(); /* barrier before setting valid flag */
1803 /* re-write the last 32-bits with the valid flags */
1804 src->flags = last_flags;
1805 src_ints = (uint32_t *)src;
1807 dst_ints = (volatile uint32_t *)dst;
1809 *dst_ints = *src_ints;
1817 mxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m,
1818 int busdma_seg_cnt, int ip_off)
1821 mcp_kreq_ether_send_t *req;
1822 bus_dma_segment_t *seg;
1825 uint32_t low, high_swapped;
1826 int len, seglen, cum_len, cum_len_next;
1827 int next_is_first, chop, cnt, rdma_count, small;
1828 uint16_t pseudo_hdr_offset, cksum_offset, mss;
1829 uint8_t flags, flags_next;
1832 mss = m->m_pkthdr.tso_segsz;
1834 /* negative cum_len signifies to the
1835 * send loop that we are still in the
1836 * header portion of the TSO packet.
1839 /* ensure we have the ethernet, IP and TCP
1840 header together in the first mbuf, copy
1841 it to a scratch buffer if not */
1842 if (__predict_false(m->m_len < ip_off + sizeof (*ip))) {
1843 m_copydata(m, 0, ip_off + sizeof (*ip),
1845 ip = (struct ip *)(ss->scratch + ip_off);
1847 ip = (struct ip *)(mtod(m, char *) + ip_off);
1849 if (__predict_false(m->m_len < ip_off + (ip->ip_hl << 2)
1851 m_copydata(m, 0, ip_off + (ip->ip_hl << 2)
1852 + sizeof (*tcp), ss->scratch);
1853 ip = (struct ip *)(mtod(m, char *) + ip_off);
1856 tcp = (struct tcphdr *)((char *)ip + (ip->ip_hl << 2));
1857 cum_len = -(ip_off + ((ip->ip_hl + tcp->th_off) << 2));
1858 cksum_offset = ip_off + (ip->ip_hl << 2);
1860 /* TSO implies checksum offload on this hardware */
1861 if (__predict_false((m->m_pkthdr.csum_flags & (CSUM_TCP)) == 0)) {
1863 * If packet has full TCP csum, replace it with pseudo hdr
1864 * sum that the NIC expects, otherwise the NIC will emit
1865 * packets with bad TCP checksums.
1867 m->m_pkthdr.csum_flags = CSUM_TCP;
1868 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1869 tcp->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
1870 htons(IPPROTO_TCP + (m->m_pkthdr.len - cksum_offset)));
1872 flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST;
1875 /* for TSO, pseudo_hdr_offset holds mss.
1876 * The firmware figures out where to put
1877 * the checksum by parsing the header. */
1878 pseudo_hdr_offset = htobe16(mss);
1885 /* "rdma_count" is the number of RDMAs belonging to the
1886 * current packet BEFORE the current send request. For
1887 * non-TSO packets, this is equal to "count".
1888 * For TSO packets, rdma_count needs to be reset
1889 * to 0 after a segment cut.
1891 * The rdma_count field of the send request is
1892 * the number of RDMAs of the packet starting at
1893 * that request. For TSO send requests with one ore more cuts
1894 * in the middle, this is the number of RDMAs starting
1895 * after the last cut in the request. All previous
1896 * segments before the last cut implicitly have 1 RDMA.
1898 * Since the number of RDMAs is not known beforehand,
1899 * it must be filled-in retroactively - after each
1900 * segmentation cut or at the end of the entire packet.
1903 while (busdma_seg_cnt) {
1904 /* Break the busdma segment up into pieces*/
1905 low = MXGE_LOWPART_TO_U32(seg->ds_addr);
1906 high_swapped = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
1910 flags_next = flags & ~MXGEFW_FLAGS_FIRST;
1912 cum_len_next = cum_len + seglen;
1913 (req-rdma_count)->rdma_count = rdma_count + 1;
1914 if (__predict_true(cum_len >= 0)) {
1916 chop = (cum_len_next > mss);
1917 cum_len_next = cum_len_next % mss;
1918 next_is_first = (cum_len_next == 0);
1919 flags |= chop * MXGEFW_FLAGS_TSO_CHOP;
1920 flags_next |= next_is_first *
1922 rdma_count |= -(chop | next_is_first);
1923 rdma_count += chop & !next_is_first;
1924 } else if (cum_len_next >= 0) {
1929 small = (mss <= MXGEFW_SEND_SMALL_SIZE);
1930 flags_next = MXGEFW_FLAGS_TSO_PLD |
1931 MXGEFW_FLAGS_FIRST |
1932 (small * MXGEFW_FLAGS_SMALL);
1935 req->addr_high = high_swapped;
1936 req->addr_low = htobe32(low);
1937 req->pseudo_hdr_offset = pseudo_hdr_offset;
1939 req->rdma_count = 1;
1940 req->length = htobe16(seglen);
1941 req->cksum_offset = cksum_offset;
1942 req->flags = flags | ((cum_len & 1) *
1943 MXGEFW_FLAGS_ALIGN_ODD);
1946 cum_len = cum_len_next;
1951 if (__predict_false(cksum_offset > seglen))
1952 cksum_offset -= seglen;
1955 if (__predict_false(cnt > tx->max_desc))
1961 (req-rdma_count)->rdma_count = rdma_count;
1965 req->flags |= MXGEFW_FLAGS_TSO_LAST;
1966 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST)));
1968 tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
1969 mxge_submit_req(tx, tx->req_list, cnt);
1970 #ifdef IFNET_BUF_RING
1971 if ((ss->sc->num_slices > 1) && tx->queue_active == 0) {
1972 /* tell the NIC to start polling this slice */
1974 tx->queue_active = 1;
1982 bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map);
1986 printf("tx->max_desc exceeded via TSO!\n");
1987 printf("mss = %d, %ld, %d!\n", mss,
1988 (long)seg - (long)tx->seg_list, tx->max_desc);
1995 #endif /* IFCAP_TSO4 */
1997 #ifdef MXGE_NEW_VLAN_API
1999 * We reproduce the software vlan tag insertion from
2000 * net/if_vlan.c:vlan_start() here so that we can advertise "hardware"
2001 * vlan tag insertion. We need to advertise this in order to have the
2002 * vlan interface respect our csum offload flags.
2004 static struct mbuf *
2005 mxge_vlan_tag_insert(struct mbuf *m)
2007 struct ether_vlan_header *evl;
2009 M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_DONTWAIT);
2010 if (__predict_false(m == NULL))
2012 if (m->m_len < sizeof(*evl)) {
2013 m = m_pullup(m, sizeof(*evl));
2014 if (__predict_false(m == NULL))
2018 * Transform the Ethernet header into an Ethernet header
2019 * with 802.1Q encapsulation.
2021 evl = mtod(m, struct ether_vlan_header *);
2022 bcopy((char *)evl + ETHER_VLAN_ENCAP_LEN,
2023 (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN);
2024 evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
2025 evl->evl_tag = htons(m->m_pkthdr.ether_vtag);
2026 m->m_flags &= ~M_VLANTAG;
2029 #endif /* MXGE_NEW_VLAN_API */
2032 mxge_encap(struct mxge_slice_state *ss, struct mbuf *m)
2035 mcp_kreq_ether_send_t *req;
2036 bus_dma_segment_t *seg;
2041 int cnt, cum_len, err, i, idx, odd_flag, ip_off;
2042 uint16_t pseudo_hdr_offset;
2043 uint8_t flags, cksum_offset;
2050 ip_off = sizeof (struct ether_header);
2051 #ifdef MXGE_NEW_VLAN_API
2052 if (m->m_flags & M_VLANTAG) {
2053 m = mxge_vlan_tag_insert(m);
2054 if (__predict_false(m == NULL))
2056 ip_off += ETHER_VLAN_ENCAP_LEN;
2059 /* (try to) map the frame for DMA */
2060 idx = tx->req & tx->mask;
2061 err = bus_dmamap_load_mbuf_sg(tx->dmat, tx->info[idx].map,
2062 m, tx->seg_list, &cnt,
2064 if (__predict_false(err == EFBIG)) {
2065 /* Too many segments in the chain. Try
2067 m_tmp = m_defrag(m, M_NOWAIT);
2068 if (m_tmp == NULL) {
2073 err = bus_dmamap_load_mbuf_sg(tx->dmat,
2075 m, tx->seg_list, &cnt,
2078 if (__predict_false(err != 0)) {
2079 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d"
2080 " packet len = %d\n", err, m->m_pkthdr.len);
2083 bus_dmamap_sync(tx->dmat, tx->info[idx].map,
2084 BUS_DMASYNC_PREWRITE);
2085 tx->info[idx].m = m;
2088 /* TSO is different enough, we handle it in another routine */
2089 if (m->m_pkthdr.csum_flags & (CSUM_TSO)) {
2090 mxge_encap_tso(ss, m, cnt, ip_off);
2097 pseudo_hdr_offset = 0;
2098 flags = MXGEFW_FLAGS_NO_TSO;
2100 /* checksum offloading? */
2101 if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA)) {
2102 /* ensure ip header is in first mbuf, copy
2103 it to a scratch buffer if not */
2104 if (__predict_false(m->m_len < ip_off + sizeof (*ip))) {
2105 m_copydata(m, 0, ip_off + sizeof (*ip),
2107 ip = (struct ip *)(ss->scratch + ip_off);
2109 ip = (struct ip *)(mtod(m, char *) + ip_off);
2111 cksum_offset = ip_off + (ip->ip_hl << 2);
2112 pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data;
2113 pseudo_hdr_offset = htobe16(pseudo_hdr_offset);
2114 req->cksum_offset = cksum_offset;
2115 flags |= MXGEFW_FLAGS_CKSUM;
2116 odd_flag = MXGEFW_FLAGS_ALIGN_ODD;
2120 if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE)
2121 flags |= MXGEFW_FLAGS_SMALL;
2123 /* convert segments into a request list */
2126 req->flags = MXGEFW_FLAGS_FIRST;
2127 for (i = 0; i < cnt; i++) {
2129 htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr));
2131 htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
2132 req->length = htobe16(seg->ds_len);
2133 req->cksum_offset = cksum_offset;
2134 if (cksum_offset > seg->ds_len)
2135 cksum_offset -= seg->ds_len;
2138 req->pseudo_hdr_offset = pseudo_hdr_offset;
2139 req->pad = 0; /* complete solid 16-byte block */
2140 req->rdma_count = 1;
2141 req->flags |= flags | ((cum_len & 1) * odd_flag);
2142 cum_len += seg->ds_len;
2148 /* pad runts to 60 bytes */
2152 htobe32(MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr));
2154 htobe32(MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr));
2155 req->length = htobe16(60 - cum_len);
2156 req->cksum_offset = 0;
2157 req->pseudo_hdr_offset = pseudo_hdr_offset;
2158 req->pad = 0; /* complete solid 16-byte block */
2159 req->rdma_count = 1;
2160 req->flags |= flags | ((cum_len & 1) * odd_flag);
2164 tx->req_list[0].rdma_count = cnt;
2166 /* print what the firmware will see */
2167 for (i = 0; i < cnt; i++) {
2168 printf("%d: addr: 0x%x 0x%x len:%d pso%d,"
2169 "cso:%d, flags:0x%x, rdma:%d\n",
2170 i, (int)ntohl(tx->req_list[i].addr_high),
2171 (int)ntohl(tx->req_list[i].addr_low),
2172 (int)ntohs(tx->req_list[i].length),
2173 (int)ntohs(tx->req_list[i].pseudo_hdr_offset),
2174 tx->req_list[i].cksum_offset, tx->req_list[i].flags,
2175 tx->req_list[i].rdma_count);
2177 printf("--------------\n");
2179 tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
2180 mxge_submit_req(tx, tx->req_list, cnt);
2181 #ifdef IFNET_BUF_RING
2182 if ((ss->sc->num_slices > 1) && tx->queue_active == 0) {
2183 /* tell the NIC to start polling this slice */
2185 tx->queue_active = 1;
2198 #ifdef IFNET_BUF_RING
2200 mxge_qflush(struct ifnet *ifp)
2202 mxge_softc_t *sc = ifp->if_softc;
2207 for (slice = 0; slice < sc->num_slices; slice++) {
2208 tx = &sc->ss[slice].tx;
2210 while ((m = buf_ring_dequeue_sc(tx->br)) != NULL)
2212 mtx_unlock(&tx->mtx);
2218 mxge_start_locked(struct mxge_slice_state *ss)
2229 while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) {
2230 m = drbr_dequeue(ifp, tx->br);
2234 /* let BPF see it */
2237 /* give it to the nic */
2240 /* ran out of transmit slots */
2241 if (((ss->if_drv_flags & IFF_DRV_OACTIVE) == 0)
2242 && (!drbr_empty(ifp, tx->br))) {
2243 ss->if_drv_flags |= IFF_DRV_OACTIVE;
2249 mxge_transmit_locked(struct mxge_slice_state *ss, struct mbuf *m)
2260 if ((ss->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
2262 err = drbr_enqueue(ifp, tx->br, m);
2266 if (!drbr_needs_enqueue(ifp, tx->br) &&
2267 ((tx->mask - (tx->req - tx->done)) > tx->max_desc)) {
2268 /* let BPF see it */
2270 /* give it to the nic */
2272 } else if ((err = drbr_enqueue(ifp, tx->br, m)) != 0) {
2275 if (!drbr_empty(ifp, tx->br))
2276 mxge_start_locked(ss);
2281 mxge_transmit(struct ifnet *ifp, struct mbuf *m)
2283 mxge_softc_t *sc = ifp->if_softc;
2284 struct mxge_slice_state *ss;
2289 slice = m->m_pkthdr.flowid;
2290 slice &= (sc->num_slices - 1); /* num_slices always power of 2 */
2292 ss = &sc->ss[slice];
2295 if (mtx_trylock(&tx->mtx)) {
2296 err = mxge_transmit_locked(ss, m);
2297 mtx_unlock(&tx->mtx);
2299 err = drbr_enqueue(ifp, tx->br, m);
2308 mxge_start_locked(struct mxge_slice_state *ss)
2318 while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) {
2319 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
2323 /* let BPF see it */
2326 /* give it to the nic */
2329 /* ran out of transmit slots */
2330 if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
2331 sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2337 mxge_start(struct ifnet *ifp)
2339 mxge_softc_t *sc = ifp->if_softc;
2340 struct mxge_slice_state *ss;
2342 /* only use the first slice for now */
2344 mtx_lock(&ss->tx.mtx);
2345 mxge_start_locked(ss);
2346 mtx_unlock(&ss->tx.mtx);
2350 * copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy
2351 * at most 32 bytes at a time, so as to avoid involving the software
2352 * pio handler in the nic. We re-write the first segment's low
2353 * DMA address to mark it valid only after we write the entire chunk
2357 mxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst,
2358 mcp_kreq_ether_recv_t *src)
2362 low = src->addr_low;
2363 src->addr_low = 0xffffffff;
2364 mxge_pio_copy(dst, src, 4 * sizeof (*src));
2366 mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
2368 src->addr_low = low;
2369 dst->addr_low = low;
2374 mxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2376 bus_dma_segment_t seg;
2378 mxge_rx_ring_t *rx = &ss->rx_small;
2381 m = m_gethdr(M_DONTWAIT, MT_DATA);
2388 err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m,
2389 &seg, &cnt, BUS_DMA_NOWAIT);
2394 rx->info[idx].m = m;
2395 rx->shadow[idx].addr_low =
2396 htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr));
2397 rx->shadow[idx].addr_high =
2398 htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr));
2402 mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]);
2407 mxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2409 bus_dma_segment_t seg[3];
2411 mxge_rx_ring_t *rx = &ss->rx_big;
2414 if (rx->cl_size == MCLBYTES)
2415 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
2417 m = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, rx->cl_size);
2423 m->m_len = rx->mlen;
2424 err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m,
2425 seg, &cnt, BUS_DMA_NOWAIT);
2430 rx->info[idx].m = m;
2431 rx->shadow[idx].addr_low =
2432 htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr));
2433 rx->shadow[idx].addr_high =
2434 htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
2436 #if MXGE_VIRT_JUMBOS
2437 for (i = 1; i < cnt; i++) {
2438 rx->shadow[idx + i].addr_low =
2439 htobe32(MXGE_LOWPART_TO_U32(seg[i].ds_addr));
2440 rx->shadow[idx + i].addr_high =
2441 htobe32(MXGE_HIGHPART_TO_U32(seg[i].ds_addr));
2446 for (i = 0; i < rx->nbufs; i++) {
2447 if ((idx & 7) == 7) {
2448 mxge_submit_8rx(&rx->lanai[idx - 7],
2449 &rx->shadow[idx - 7]);
2457 * Myri10GE hardware checksums are not valid if the sender
2458 * padded the frame with non-zero padding. This is because
2459 * the firmware just does a simple 16-bit 1s complement
2460 * checksum across the entire frame, excluding the first 14
2461 * bytes. It is best to simply to check the checksum and
2462 * tell the stack about it only if the checksum is good
2465 static inline uint16_t
2466 mxge_rx_csum(struct mbuf *m, int csum)
2468 struct ether_header *eh;
2472 eh = mtod(m, struct ether_header *);
2474 /* only deal with IPv4 TCP & UDP for now */
2475 if (__predict_false(eh->ether_type != htons(ETHERTYPE_IP)))
2477 ip = (struct ip *)(eh + 1);
2478 if (__predict_false(ip->ip_p != IPPROTO_TCP &&
2479 ip->ip_p != IPPROTO_UDP))
2482 c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2483 htonl(ntohs(csum) + ntohs(ip->ip_len) +
2484 - (ip->ip_hl << 2) + ip->ip_p));
2493 mxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum)
2495 struct ether_vlan_header *evl;
2496 struct ether_header *eh;
2499 evl = mtod(m, struct ether_vlan_header *);
2500 eh = mtod(m, struct ether_header *);
2503 * fix checksum by subtracting ETHER_VLAN_ENCAP_LEN bytes
2504 * after what the firmware thought was the end of the ethernet
2508 /* put checksum into host byte order */
2509 *csum = ntohs(*csum);
2510 partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN));
2511 (*csum) += ~partial;
2512 (*csum) += ((*csum) < ~partial);
2513 (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2514 (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2516 /* restore checksum to network byte order;
2517 later consumers expect this */
2518 *csum = htons(*csum);
2521 #ifdef MXGE_NEW_VLAN_API
2522 m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag);
2526 mtag = m_tag_alloc(MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int),
2530 VLAN_TAG_VALUE(mtag) = ntohs(evl->evl_tag);
2531 m_tag_prepend(m, mtag);
2535 m->m_flags |= M_VLANTAG;
2538 * Remove the 802.1q header by copying the Ethernet
2539 * addresses over it and adjusting the beginning of
2540 * the data in the mbuf. The encapsulated Ethernet
2541 * type field is already in place.
2543 bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN,
2544 ETHER_HDR_LEN - ETHER_TYPE_LEN);
2545 m_adj(m, ETHER_VLAN_ENCAP_LEN);
2550 mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, uint32_t csum)
2555 struct ether_header *eh;
2557 bus_dmamap_t old_map;
2559 uint16_t tcpudp_csum;
2564 idx = rx->cnt & rx->mask;
2565 rx->cnt += rx->nbufs;
2566 /* save a pointer to the received mbuf */
2567 m = rx->info[idx].m;
2568 /* try to replace the received mbuf */
2569 if (mxge_get_buf_big(ss, rx->extra_map, idx)) {
2570 /* drop the frame -- the old mbuf is re-cycled */
2575 /* unmap the received buffer */
2576 old_map = rx->info[idx].map;
2577 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2578 bus_dmamap_unload(rx->dmat, old_map);
2580 /* swap the bus_dmamap_t's */
2581 rx->info[idx].map = rx->extra_map;
2582 rx->extra_map = old_map;
2584 /* mcp implicitly skips 1st 2 bytes so that packet is properly
2586 m->m_data += MXGEFW_PAD;
2588 m->m_pkthdr.rcvif = ifp;
2589 m->m_len = m->m_pkthdr.len = len;
2591 eh = mtod(m, struct ether_header *);
2592 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2593 mxge_vlan_tag_remove(m, &csum);
2595 /* if the checksum is valid, mark it in the mbuf header */
2596 if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) {
2597 if (sc->lro_cnt && (0 == mxge_lro_rx(ss, m, csum)))
2599 /* otherwise, it was a UDP frame, or a TCP frame which
2600 we could not do LRO on. Tell the stack that the
2602 m->m_pkthdr.csum_data = 0xffff;
2603 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID;
2605 /* flowid only valid if RSS hashing is enabled */
2606 if (sc->num_slices > 1) {
2607 m->m_pkthdr.flowid = (ss - sc->ss);
2608 m->m_flags |= M_FLOWID;
2610 /* pass the frame up the stack */
2611 (*ifp->if_input)(ifp, m);
2615 mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, uint32_t csum)
2619 struct ether_header *eh;
2622 bus_dmamap_t old_map;
2624 uint16_t tcpudp_csum;
2629 idx = rx->cnt & rx->mask;
2631 /* save a pointer to the received mbuf */
2632 m = rx->info[idx].m;
2633 /* try to replace the received mbuf */
2634 if (mxge_get_buf_small(ss, rx->extra_map, idx)) {
2635 /* drop the frame -- the old mbuf is re-cycled */
2640 /* unmap the received buffer */
2641 old_map = rx->info[idx].map;
2642 bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2643 bus_dmamap_unload(rx->dmat, old_map);
2645 /* swap the bus_dmamap_t's */
2646 rx->info[idx].map = rx->extra_map;
2647 rx->extra_map = old_map;
2649 /* mcp implicitly skips 1st 2 bytes so that packet is properly
2651 m->m_data += MXGEFW_PAD;
2653 m->m_pkthdr.rcvif = ifp;
2654 m->m_len = m->m_pkthdr.len = len;
2656 eh = mtod(m, struct ether_header *);
2657 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2658 mxge_vlan_tag_remove(m, &csum);
2660 /* if the checksum is valid, mark it in the mbuf header */
2661 if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) {
2662 if (sc->lro_cnt && (0 == mxge_lro_rx(ss, m, csum)))
2664 /* otherwise, it was a UDP frame, or a TCP frame which
2665 we could not do LRO on. Tell the stack that the
2667 m->m_pkthdr.csum_data = 0xffff;
2668 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID;
2670 /* flowid only valid if RSS hashing is enabled */
2671 if (sc->num_slices > 1) {
2672 m->m_pkthdr.flowid = (ss - sc->ss);
2673 m->m_flags |= M_FLOWID;
2675 /* pass the frame up the stack */
2676 (*ifp->if_input)(ifp, m);
2680 mxge_clean_rx_done(struct mxge_slice_state *ss)
2682 mxge_rx_done_t *rx_done = &ss->rx_done;
2688 while (rx_done->entry[rx_done->idx].length != 0) {
2689 length = ntohs(rx_done->entry[rx_done->idx].length);
2690 rx_done->entry[rx_done->idx].length = 0;
2691 checksum = rx_done->entry[rx_done->idx].checksum;
2692 if (length <= (MHLEN - MXGEFW_PAD))
2693 mxge_rx_done_small(ss, length, checksum);
2695 mxge_rx_done_big(ss, length, checksum);
2697 rx_done->idx = rx_done->cnt & rx_done->mask;
2699 /* limit potential for livelock */
2700 if (__predict_false(++limit > rx_done->mask / 2))
2704 while (!SLIST_EMPTY(&ss->lro_active)) {
2705 struct lro_entry *lro = SLIST_FIRST(&ss->lro_active);
2706 SLIST_REMOVE_HEAD(&ss->lro_active, next);
2707 mxge_lro_flush(ss, lro);
2714 mxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx)
2725 while (tx->pkt_done != mcp_idx) {
2726 idx = tx->done & tx->mask;
2728 m = tx->info[idx].m;
2729 /* mbuf and DMA map only attached to the first
2732 ss->obytes += m->m_pkthdr.len;
2733 if (m->m_flags & M_MCAST)
2736 tx->info[idx].m = NULL;
2737 map = tx->info[idx].map;
2738 bus_dmamap_unload(tx->dmat, map);
2741 if (tx->info[idx].flag) {
2742 tx->info[idx].flag = 0;
2747 /* If we have space, clear IFF_OACTIVE to tell the stack that
2748 its OK to send packets */
2749 #ifdef IFNET_BUF_RING
2750 flags = &ss->if_drv_flags;
2752 flags = &ifp->if_drv_flags;
2754 mtx_lock(&ss->tx.mtx);
2755 if ((*flags) & IFF_DRV_OACTIVE &&
2756 tx->req - tx->done < (tx->mask + 1)/4) {
2757 *(flags) &= ~IFF_DRV_OACTIVE;
2759 mxge_start_locked(ss);
2761 #ifdef IFNET_BUF_RING
2762 if ((ss->sc->num_slices > 1) && (tx->req == tx->done)) {
2763 /* let the NIC stop polling this queue, since there
2764 * are no more transmits pending */
2765 if (tx->req == tx->done) {
2767 tx->queue_active = 0;
2773 mtx_unlock(&ss->tx.mtx);
2777 static struct mxge_media_type mxge_xfp_media_types[] =
2779 {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"},
2780 {IFM_10G_SR, (1 << 7), "10GBASE-SR"},
2781 {IFM_10G_LR, (1 << 6), "10GBASE-LR"},
2782 {0, (1 << 5), "10GBASE-ER"},
2783 {IFM_10G_LRM, (1 << 4), "10GBASE-LRM"},
2784 {0, (1 << 3), "10GBASE-SW"},
2785 {0, (1 << 2), "10GBASE-LW"},
2786 {0, (1 << 1), "10GBASE-EW"},
2787 {0, (1 << 0), "Reserved"}
2789 static struct mxge_media_type mxge_sfp_media_types[] =
2791 {IFM_10G_TWINAX, 0, "10GBASE-Twinax"},
2792 {0, (1 << 7), "Reserved"},
2793 {IFM_10G_LRM, (1 << 6), "10GBASE-LRM"},
2794 {IFM_10G_LR, (1 << 5), "10GBASE-LR"},
2795 {IFM_10G_SR, (1 << 4), "10GBASE-SR"},
2796 {IFM_10G_TWINAX,(1 << 0), "10GBASE-Twinax"}
2800 mxge_media_set(mxge_softc_t *sc, int media_type)
2804 ifmedia_add(&sc->media, IFM_ETHER | IFM_FDX | media_type,
2806 ifmedia_set(&sc->media, IFM_ETHER | IFM_FDX | media_type);
2807 sc->current_media = media_type;
2808 sc->media.ifm_media = sc->media.ifm_cur->ifm_media;
2812 mxge_media_init(mxge_softc_t *sc)
2817 ifmedia_removeall(&sc->media);
2818 mxge_media_set(sc, IFM_AUTO);
2821 * parse the product code to deterimine the interface type
2822 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character
2823 * after the 3rd dash in the driver's cached copy of the
2824 * EEPROM's product code string.
2826 ptr = sc->product_code_string;
2828 device_printf(sc->dev, "Missing product code\n");
2832 for (i = 0; i < 3; i++, ptr++) {
2833 ptr = index(ptr, '-');
2835 device_printf(sc->dev,
2836 "only %d dashes in PC?!?\n", i);
2842 sc->connector = MXGE_CX4;
2843 mxge_media_set(sc, IFM_10G_CX4);
2844 } else if (*ptr == 'Q') {
2845 /* -Q is Quad Ribbon Fiber */
2846 sc->connector = MXGE_QRF;
2847 device_printf(sc->dev, "Quad Ribbon Fiber Media\n");
2848 /* FreeBSD has no media type for Quad ribbon fiber */
2849 } else if (*ptr == 'R') {
2851 sc->connector = MXGE_XFP;
2852 } else if (*ptr == 'S' || *(ptr +1) == 'S') {
2853 /* -S or -2S is SFP+ */
2854 sc->connector = MXGE_SFP;
2856 device_printf(sc->dev, "Unknown media type: %c\n", *ptr);
2861 * Determine the media type for a NIC. Some XFPs will identify
2862 * themselves only when their link is up, so this is initiated via a
2863 * link up interrupt. However, this can potentially take up to
2864 * several milliseconds, so it is run via the watchdog routine, rather
2865 * than in the interrupt handler itself.
2868 mxge_media_probe(mxge_softc_t *sc)
2873 struct mxge_media_type *mxge_media_types = NULL;
2874 int i, err, ms, mxge_media_type_entries;
2877 sc->need_media_probe = 0;
2879 if (sc->connector == MXGE_XFP) {
2881 mxge_media_types = mxge_xfp_media_types;
2882 mxge_media_type_entries =
2883 sizeof (mxge_xfp_media_types) /
2884 sizeof (mxge_xfp_media_types[0]);
2885 byte = MXGE_XFP_COMPLIANCE_BYTE;
2887 } else if (sc->connector == MXGE_SFP) {
2888 /* -S or -2S is SFP+ */
2889 mxge_media_types = mxge_sfp_media_types;
2890 mxge_media_type_entries =
2891 sizeof (mxge_sfp_media_types) /
2892 sizeof (mxge_sfp_media_types[0]);
2896 /* nothing to do; media type cannot change */
2901 * At this point we know the NIC has an XFP cage, so now we
2902 * try to determine what is in the cage by using the
2903 * firmware's XFP I2C commands to read the XFP 10GbE compilance
2904 * register. We read just one byte, which may take over
2908 cmd.data0 = 0; /* just fetch 1 byte, not all 256 */
2910 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd);
2911 if (err == MXGEFW_CMD_ERROR_I2C_FAILURE) {
2912 device_printf(sc->dev, "failed to read XFP\n");
2914 if (err == MXGEFW_CMD_ERROR_I2C_ABSENT) {
2915 device_printf(sc->dev, "Type R/S with no XFP!?!?\n");
2917 if (err != MXGEFW_CMD_OK) {
2921 /* now we wait for the data to be cached */
2923 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
2924 for (ms = 0; (err == EBUSY) && (ms < 50); ms++) {
2927 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
2929 if (err != MXGEFW_CMD_OK) {
2930 device_printf(sc->dev, "failed to read %s (%d, %dms)\n",
2931 cage_type, err, ms);
2935 if (cmd.data0 == mxge_media_types[0].bitmask) {
2937 device_printf(sc->dev, "%s:%s\n", cage_type,
2938 mxge_media_types[0].name);
2939 if (sc->current_media != mxge_media_types[0].flag) {
2940 mxge_media_init(sc);
2941 mxge_media_set(sc, mxge_media_types[0].flag);
2945 for (i = 1; i < mxge_media_type_entries; i++) {
2946 if (cmd.data0 & mxge_media_types[i].bitmask) {
2948 device_printf(sc->dev, "%s:%s\n",
2950 mxge_media_types[i].name);
2952 if (sc->current_media != mxge_media_types[i].flag) {
2953 mxge_media_init(sc);
2954 mxge_media_set(sc, mxge_media_types[i].flag);
2960 device_printf(sc->dev, "%s media 0x%x unknown\n",
2961 cage_type, cmd.data0);
2967 mxge_intr(void *arg)
2969 struct mxge_slice_state *ss = arg;
2970 mxge_softc_t *sc = ss->sc;
2971 mcp_irq_data_t *stats = ss->fw_stats;
2972 mxge_tx_ring_t *tx = &ss->tx;
2973 mxge_rx_done_t *rx_done = &ss->rx_done;
2974 uint32_t send_done_count;
2978 #ifndef IFNET_BUF_RING
2979 /* an interrupt on a non-zero slice is implicitly valid
2980 since MSI-X irqs are not shared */
2982 mxge_clean_rx_done(ss);
2983 *ss->irq_claim = be32toh(3);
2988 /* make sure the DMA has finished */
2989 if (!stats->valid) {
2992 valid = stats->valid;
2994 if (sc->legacy_irq) {
2995 /* lower legacy IRQ */
2996 *sc->irq_deassert = 0;
2997 if (!mxge_deassert_wait)
2998 /* don't wait for conf. that irq is low */
3004 /* loop while waiting for legacy irq deassertion */
3006 /* check for transmit completes and receives */
3007 send_done_count = be32toh(stats->send_done_count);
3008 while ((send_done_count != tx->pkt_done) ||
3009 (rx_done->entry[rx_done->idx].length != 0)) {
3010 if (send_done_count != tx->pkt_done)
3011 mxge_tx_done(ss, (int)send_done_count);
3012 mxge_clean_rx_done(ss);
3013 send_done_count = be32toh(stats->send_done_count);
3015 if (sc->legacy_irq && mxge_deassert_wait)
3017 } while (*((volatile uint8_t *) &stats->valid));
3019 /* fw link & error stats meaningful only on the first slice */
3020 if (__predict_false((ss == sc->ss) && stats->stats_updated)) {
3021 if (sc->link_state != stats->link_up) {
3022 sc->link_state = stats->link_up;
3023 if (sc->link_state) {
3024 if_link_state_change(sc->ifp, LINK_STATE_UP);
3025 sc->ifp->if_baudrate = IF_Gbps(10UL);
3027 device_printf(sc->dev, "link up\n");
3029 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
3030 sc->ifp->if_baudrate = 0;
3032 device_printf(sc->dev, "link down\n");
3034 sc->need_media_probe = 1;
3036 if (sc->rdma_tags_available !=
3037 be32toh(stats->rdma_tags_available)) {
3038 sc->rdma_tags_available =
3039 be32toh(stats->rdma_tags_available);
3040 device_printf(sc->dev, "RDMA timed out! %d tags "
3041 "left\n", sc->rdma_tags_available);
3044 if (stats->link_down) {
3045 sc->down_cnt += stats->link_down;
3047 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
3051 /* check to see if we have rx token to pass back */
3053 *ss->irq_claim = be32toh(3);
3054 *(ss->irq_claim + 1) = be32toh(3);
3058 mxge_init(void *arg)
3065 mxge_free_slice_mbufs(struct mxge_slice_state *ss)
3067 struct lro_entry *lro_entry;
3070 while (!SLIST_EMPTY(&ss->lro_free)) {
3071 lro_entry = SLIST_FIRST(&ss->lro_free);
3072 SLIST_REMOVE_HEAD(&ss->lro_free, next);
3073 free(lro_entry, M_DEVBUF);
3076 for (i = 0; i <= ss->rx_big.mask; i++) {
3077 if (ss->rx_big.info[i].m == NULL)
3079 bus_dmamap_unload(ss->rx_big.dmat,
3080 ss->rx_big.info[i].map);
3081 m_freem(ss->rx_big.info[i].m);
3082 ss->rx_big.info[i].m = NULL;
3085 for (i = 0; i <= ss->rx_small.mask; i++) {
3086 if (ss->rx_small.info[i].m == NULL)
3088 bus_dmamap_unload(ss->rx_small.dmat,
3089 ss->rx_small.info[i].map);
3090 m_freem(ss->rx_small.info[i].m);
3091 ss->rx_small.info[i].m = NULL;
3094 /* transmit ring used only on the first slice */
3095 if (ss->tx.info == NULL)
3098 for (i = 0; i <= ss->tx.mask; i++) {
3099 ss->tx.info[i].flag = 0;
3100 if (ss->tx.info[i].m == NULL)
3102 bus_dmamap_unload(ss->tx.dmat,
3103 ss->tx.info[i].map);
3104 m_freem(ss->tx.info[i].m);
3105 ss->tx.info[i].m = NULL;
3110 mxge_free_mbufs(mxge_softc_t *sc)
3114 for (slice = 0; slice < sc->num_slices; slice++)
3115 mxge_free_slice_mbufs(&sc->ss[slice]);
3119 mxge_free_slice_rings(struct mxge_slice_state *ss)
3124 if (ss->rx_done.entry != NULL)
3125 mxge_dma_free(&ss->rx_done.dma);
3126 ss->rx_done.entry = NULL;
3128 if (ss->tx.req_bytes != NULL)
3129 free(ss->tx.req_bytes, M_DEVBUF);
3130 ss->tx.req_bytes = NULL;
3132 if (ss->tx.seg_list != NULL)
3133 free(ss->tx.seg_list, M_DEVBUF);
3134 ss->tx.seg_list = NULL;
3136 if (ss->rx_small.shadow != NULL)
3137 free(ss->rx_small.shadow, M_DEVBUF);
3138 ss->rx_small.shadow = NULL;
3140 if (ss->rx_big.shadow != NULL)
3141 free(ss->rx_big.shadow, M_DEVBUF);
3142 ss->rx_big.shadow = NULL;
3144 if (ss->tx.info != NULL) {
3145 if (ss->tx.dmat != NULL) {
3146 for (i = 0; i <= ss->tx.mask; i++) {
3147 bus_dmamap_destroy(ss->tx.dmat,
3148 ss->tx.info[i].map);
3150 bus_dma_tag_destroy(ss->tx.dmat);
3152 free(ss->tx.info, M_DEVBUF);
3156 if (ss->rx_small.info != NULL) {
3157 if (ss->rx_small.dmat != NULL) {
3158 for (i = 0; i <= ss->rx_small.mask; i++) {
3159 bus_dmamap_destroy(ss->rx_small.dmat,
3160 ss->rx_small.info[i].map);
3162 bus_dmamap_destroy(ss->rx_small.dmat,
3163 ss->rx_small.extra_map);
3164 bus_dma_tag_destroy(ss->rx_small.dmat);
3166 free(ss->rx_small.info, M_DEVBUF);
3168 ss->rx_small.info = NULL;
3170 if (ss->rx_big.info != NULL) {
3171 if (ss->rx_big.dmat != NULL) {
3172 for (i = 0; i <= ss->rx_big.mask; i++) {
3173 bus_dmamap_destroy(ss->rx_big.dmat,
3174 ss->rx_big.info[i].map);
3176 bus_dmamap_destroy(ss->rx_big.dmat,
3177 ss->rx_big.extra_map);
3178 bus_dma_tag_destroy(ss->rx_big.dmat);
3180 free(ss->rx_big.info, M_DEVBUF);
3182 ss->rx_big.info = NULL;
3186 mxge_free_rings(mxge_softc_t *sc)
3190 for (slice = 0; slice < sc->num_slices; slice++)
3191 mxge_free_slice_rings(&sc->ss[slice]);
3195 mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries,
3196 int tx_ring_entries)
3198 mxge_softc_t *sc = ss->sc;
3204 /* allocate per-slice receive resources */
3206 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
3207 ss->rx_done.mask = (2 * rx_ring_entries) - 1;
3209 /* allocate the rx shadow rings */
3210 bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
3211 ss->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3212 if (ss->rx_small.shadow == NULL)
3215 bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
3216 ss->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3217 if (ss->rx_big.shadow == NULL)
3220 /* allocate the rx host info rings */
3221 bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
3222 ss->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3223 if (ss->rx_small.info == NULL)
3226 bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
3227 ss->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3228 if (ss->rx_big.info == NULL)
3231 /* allocate the rx busdma resources */
3232 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
3234 4096, /* boundary */
3235 BUS_SPACE_MAXADDR, /* low */
3236 BUS_SPACE_MAXADDR, /* high */
3237 NULL, NULL, /* filter */
3238 MHLEN, /* maxsize */
3240 MHLEN, /* maxsegsize */
3241 BUS_DMA_ALLOCNOW, /* flags */
3242 NULL, NULL, /* lock */
3243 &ss->rx_small.dmat); /* tag */
3245 device_printf(sc->dev, "Err %d allocating rx_small dmat\n",
3250 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
3252 #if MXGE_VIRT_JUMBOS
3253 4096, /* boundary */
3257 BUS_SPACE_MAXADDR, /* low */
3258 BUS_SPACE_MAXADDR, /* high */
3259 NULL, NULL, /* filter */
3260 3*4096, /* maxsize */
3261 #if MXGE_VIRT_JUMBOS
3263 4096, /* maxsegsize*/
3266 MJUM9BYTES, /* maxsegsize*/
3268 BUS_DMA_ALLOCNOW, /* flags */
3269 NULL, NULL, /* lock */
3270 &ss->rx_big.dmat); /* tag */
3272 device_printf(sc->dev, "Err %d allocating rx_big dmat\n",
3276 for (i = 0; i <= ss->rx_small.mask; i++) {
3277 err = bus_dmamap_create(ss->rx_small.dmat, 0,
3278 &ss->rx_small.info[i].map);
3280 device_printf(sc->dev, "Err %d rx_small dmamap\n",
3285 err = bus_dmamap_create(ss->rx_small.dmat, 0,
3286 &ss->rx_small.extra_map);
3288 device_printf(sc->dev, "Err %d extra rx_small dmamap\n",
3293 for (i = 0; i <= ss->rx_big.mask; i++) {
3294 err = bus_dmamap_create(ss->rx_big.dmat, 0,
3295 &ss->rx_big.info[i].map);
3297 device_printf(sc->dev, "Err %d rx_big dmamap\n",
3302 err = bus_dmamap_create(ss->rx_big.dmat, 0,
3303 &ss->rx_big.extra_map);
3305 device_printf(sc->dev, "Err %d extra rx_big dmamap\n",
3310 /* now allocate TX resouces */
3312 #ifndef IFNET_BUF_RING
3313 /* only use a single TX ring for now */
3314 if (ss != ss->sc->ss)
3318 ss->tx.mask = tx_ring_entries - 1;
3319 ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4);
3322 /* allocate the tx request copy block */
3324 sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4);
3325 ss->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK);
3326 if (ss->tx.req_bytes == NULL)
3328 /* ensure req_list entries are aligned to 8 bytes */
3329 ss->tx.req_list = (mcp_kreq_ether_send_t *)
3330 ((unsigned long)(ss->tx.req_bytes + 7) & ~7UL);
3332 /* allocate the tx busdma segment list */
3333 bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc;
3334 ss->tx.seg_list = (bus_dma_segment_t *)
3335 malloc(bytes, M_DEVBUF, M_WAITOK);
3336 if (ss->tx.seg_list == NULL)
3339 /* allocate the tx host info ring */
3340 bytes = tx_ring_entries * sizeof (*ss->tx.info);
3341 ss->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3342 if (ss->tx.info == NULL)
3345 /* allocate the tx busdma resources */
3346 err = bus_dma_tag_create(sc->parent_dmat, /* parent */
3348 sc->tx_boundary, /* boundary */
3349 BUS_SPACE_MAXADDR, /* low */
3350 BUS_SPACE_MAXADDR, /* high */
3351 NULL, NULL, /* filter */
3352 65536 + 256, /* maxsize */
3353 ss->tx.max_desc - 2, /* num segs */
3354 sc->tx_boundary, /* maxsegsz */
3355 BUS_DMA_ALLOCNOW, /* flags */
3356 NULL, NULL, /* lock */
3357 &ss->tx.dmat); /* tag */
3360 device_printf(sc->dev, "Err %d allocating tx dmat\n",
3365 /* now use these tags to setup dmamaps for each slot
3367 for (i = 0; i <= ss->tx.mask; i++) {
3368 err = bus_dmamap_create(ss->tx.dmat, 0,
3369 &ss->tx.info[i].map);
3371 device_printf(sc->dev, "Err %d tx dmamap\n",
3381 mxge_alloc_rings(mxge_softc_t *sc)
3385 int tx_ring_entries, rx_ring_entries;
3388 /* get ring sizes */
3389 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
3390 tx_ring_size = cmd.data0;
3392 device_printf(sc->dev, "Cannot determine tx ring sizes\n");
3396 tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t);
3397 rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t);
3398 IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1);
3399 sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen;
3400 IFQ_SET_READY(&sc->ifp->if_snd);
3402 for (slice = 0; slice < sc->num_slices; slice++) {
3403 err = mxge_alloc_slice_rings(&sc->ss[slice],
3412 mxge_free_rings(sc);
3419 mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs)
3421 int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD;
3423 if (bufsize < MCLBYTES) {
3424 /* easy, everything fits in a single buffer */
3425 *big_buf_size = MCLBYTES;
3426 *cl_size = MCLBYTES;
3431 if (bufsize < MJUMPAGESIZE) {
3432 /* still easy, everything still fits in a single buffer */
3433 *big_buf_size = MJUMPAGESIZE;
3434 *cl_size = MJUMPAGESIZE;
3438 #if MXGE_VIRT_JUMBOS
3439 /* now we need to use virtually contiguous buffers */
3440 *cl_size = MJUM9BYTES;
3441 *big_buf_size = 4096;
3442 *nbufs = mtu / 4096 + 1;
3443 /* needs to be a power of two, so round up */
3447 *cl_size = MJUM9BYTES;
3448 *big_buf_size = MJUM9BYTES;
3454 mxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size)
3459 struct lro_entry *lro_entry;
3464 slice = ss - sc->ss;
3466 SLIST_INIT(&ss->lro_free);
3467 SLIST_INIT(&ss->lro_active);
3469 for (i = 0; i < sc->lro_cnt; i++) {
3470 lro_entry = (struct lro_entry *)
3471 malloc(sizeof (*lro_entry), M_DEVBUF,
3473 if (lro_entry == NULL) {
3477 SLIST_INSERT_HEAD(&ss->lro_free, lro_entry, next);
3479 /* get the lanai pointers to the send and receive rings */
3482 #ifndef IFNET_BUF_RING
3483 /* We currently only send from the first slice */
3487 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
3489 (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0);
3490 ss->tx.send_go = (volatile uint32_t *)
3491 (sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice);
3492 ss->tx.send_stop = (volatile uint32_t *)
3493 (sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice);
3494 #ifndef IFNET_BUF_RING
3498 err |= mxge_send_cmd(sc,
3499 MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
3500 ss->rx_small.lanai =
3501 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
3503 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
3505 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
3508 device_printf(sc->dev,
3509 "failed to get ring sizes or locations\n");
3513 /* stock receive rings */
3514 for (i = 0; i <= ss->rx_small.mask; i++) {
3515 map = ss->rx_small.info[i].map;
3516 err = mxge_get_buf_small(ss, map, i);
3518 device_printf(sc->dev, "alloced %d/%d smalls\n",
3519 i, ss->rx_small.mask + 1);
3523 for (i = 0; i <= ss->rx_big.mask; i++) {
3524 ss->rx_big.shadow[i].addr_low = 0xffffffff;
3525 ss->rx_big.shadow[i].addr_high = 0xffffffff;
3527 ss->rx_big.nbufs = nbufs;
3528 ss->rx_big.cl_size = cl_size;
3529 ss->rx_big.mlen = ss->sc->ifp->if_mtu + ETHER_HDR_LEN +
3530 ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD;
3531 for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) {
3532 map = ss->rx_big.info[i].map;
3533 err = mxge_get_buf_big(ss, map, i);
3535 device_printf(sc->dev, "alloced %d/%d bigs\n",
3536 i, ss->rx_big.mask + 1);
3544 mxge_open(mxge_softc_t *sc)
3547 int err, big_bytes, nbufs, slice, cl_size, i;
3549 volatile uint8_t *itable;
3550 struct mxge_slice_state *ss;
3552 /* Copy the MAC address in case it was overridden */
3553 bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN);
3555 err = mxge_reset(sc, 1);
3557 device_printf(sc->dev, "failed to reset\n");
3561 if (sc->num_slices > 1) {
3562 /* setup the indirection table */
3563 cmd.data0 = sc->num_slices;
3564 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
3567 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET,
3570 device_printf(sc->dev,
3571 "failed to setup rss tables\n");
3575 /* just enable an identity mapping */
3576 itable = sc->sram + cmd.data0;
3577 for (i = 0; i < sc->num_slices; i++)
3578 itable[i] = (uint8_t)i;
3581 cmd.data1 = mxge_rss_hash_type;
3582 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd);
3584 device_printf(sc->dev, "failed to enable slices\n");
3590 mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs);
3593 err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
3595 /* error is only meaningful if we're trying to set
3596 MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */
3597 if (err && nbufs > 1) {
3598 device_printf(sc->dev,
3599 "Failed to set alway-use-n to %d\n",
3603 /* Give the firmware the mtu and the big and small buffer
3604 sizes. The firmware wants the big buf size to be a power
3605 of two. Luckily, FreeBSD's clusters are powers of two */
3606 cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3607 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd);
3608 cmd.data0 = MHLEN - MXGEFW_PAD;
3609 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE,
3611 cmd.data0 = big_bytes;
3612 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
3615 device_printf(sc->dev, "failed to setup params\n");
3619 /* Now give him the pointer to the stats block */
3621 #ifdef IFNET_BUF_RING
3622 slice < sc->num_slices;
3627 ss = &sc->ss[slice];
3629 MXGE_LOWPART_TO_U32(ss->fw_stats_dma.bus_addr);
3631 MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.bus_addr);
3632 cmd.data2 = sizeof(struct mcp_irq_data);
3633 cmd.data2 |= (slice << 16);
3634 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
3638 bus = sc->ss->fw_stats_dma.bus_addr;
3639 bus += offsetof(struct mcp_irq_data, send_done_count);
3640 cmd.data0 = MXGE_LOWPART_TO_U32(bus);
3641 cmd.data1 = MXGE_HIGHPART_TO_U32(bus);
3642 err = mxge_send_cmd(sc,
3643 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
3645 /* Firmware cannot support multicast without STATS_DMA_V2 */
3646 sc->fw_multicast_support = 0;
3648 sc->fw_multicast_support = 1;
3652 device_printf(sc->dev, "failed to setup params\n");
3656 for (slice = 0; slice < sc->num_slices; slice++) {
3657 err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size);
3659 device_printf(sc->dev, "couldn't open slice %d\n",
3665 /* Finally, start the firmware running */
3666 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd);
3668 device_printf(sc->dev, "Couldn't bring up link\n");
3671 #ifdef IFNET_BUF_RING
3672 for (slice = 0; slice < sc->num_slices; slice++) {
3673 ss = &sc->ss[slice];
3674 ss->if_drv_flags |= IFF_DRV_RUNNING;
3675 ss->if_drv_flags &= ~IFF_DRV_OACTIVE;
3678 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
3679 sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3685 mxge_free_mbufs(sc);
3691 mxge_close(mxge_softc_t *sc, int down)
3694 int err, old_down_cnt;
3695 #ifdef IFNET_BUF_RING
3696 struct mxge_slice_state *ss;
3700 #ifdef IFNET_BUF_RING
3701 for (slice = 0; slice < sc->num_slices; slice++) {
3702 ss = &sc->ss[slice];
3703 ss->if_drv_flags &= ~IFF_DRV_RUNNING;
3706 sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3708 old_down_cnt = sc->down_cnt;
3710 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
3712 device_printf(sc->dev,
3713 "Couldn't bring down link\n");
3715 if (old_down_cnt == sc->down_cnt) {
3716 /* wait for down irq */
3717 DELAY(10 * sc->intr_coal_delay);
3720 if (old_down_cnt == sc->down_cnt) {
3721 device_printf(sc->dev, "never got down irq\n");
3724 mxge_free_mbufs(sc);
3730 mxge_setup_cfg_space(mxge_softc_t *sc)
3732 device_t dev = sc->dev;
3734 uint16_t cmd, lnk, pectl;
3736 /* find the PCIe link width and set max read request to 4KB*/
3737 if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) {
3738 lnk = pci_read_config(dev, reg + 0x12, 2);
3739 sc->link_width = (lnk >> 4) & 0x3f;
3741 if (sc->pectl == 0) {
3742 pectl = pci_read_config(dev, reg + 0x8, 2);
3743 pectl = (pectl & ~0x7000) | (5 << 12);
3744 pci_write_config(dev, reg + 0x8, pectl, 2);
3747 /* restore saved pectl after watchdog reset */
3748 pci_write_config(dev, reg + 0x8, sc->pectl, 2);
3752 /* Enable DMA and Memory space access */
3753 pci_enable_busmaster(dev);
3754 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3755 cmd |= PCIM_CMD_MEMEN;
3756 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3760 mxge_read_reboot(mxge_softc_t *sc)
3762 device_t dev = sc->dev;
3765 /* find the vendor specific offset */
3766 if (pci_find_extcap(dev, PCIY_VENDOR, &vs) != 0) {
3767 device_printf(sc->dev,
3768 "could not find vendor specific offset\n");
3769 return (uint32_t)-1;
3771 /* enable read32 mode */
3772 pci_write_config(dev, vs + 0x10, 0x3, 1);
3773 /* tell NIC which register to read */
3774 pci_write_config(dev, vs + 0x18, 0xfffffff0, 4);
3775 return (pci_read_config(dev, vs + 0x14, 4));
3779 mxge_watchdog_reset(mxge_softc_t *sc)
3781 struct pci_devinfo *dinfo;
3782 struct mxge_slice_state *ss;
3783 int err, running, s, num_tx_slices = 1;
3789 device_printf(sc->dev, "Watchdog reset!\n");
3792 * check to see if the NIC rebooted. If it did, then all of
3793 * PCI config space has been reset, and things like the
3794 * busmaster bit will be zero. If this is the case, then we
3795 * must restore PCI config space before the NIC can be used
3798 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3799 if (cmd == 0xffff) {
3801 * maybe the watchdog caught the NIC rebooting; wait
3802 * up to 100ms for it to finish. If it does not come
3803 * back, then give up
3806 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3807 if (cmd == 0xffff) {
3808 device_printf(sc->dev, "NIC disappeared!\n");
3811 if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
3812 /* print the reboot status */
3813 reboot = mxge_read_reboot(sc);
3814 device_printf(sc->dev, "NIC rebooted, status = 0x%x\n",
3816 running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING;
3820 * quiesce NIC so that TX routines will not try to
3821 * xmit after restoration of BAR
3824 /* Mark the link as down */
3825 if (sc->link_state) {
3827 if_link_state_change(sc->ifp,
3830 #ifdef IFNET_BUF_RING
3831 num_tx_slices = sc->num_slices;
3833 /* grab all TX locks to ensure no tx */
3834 for (s = 0; s < num_tx_slices; s++) {
3836 mtx_lock(&ss->tx.mtx);
3840 /* restore PCI configuration space */
3841 dinfo = device_get_ivars(sc->dev);
3842 pci_cfg_restore(sc->dev, dinfo);
3844 /* and redo any changes we made to our config space */
3845 mxge_setup_cfg_space(sc);
3848 err = mxge_load_firmware(sc, 0);
3850 device_printf(sc->dev,
3851 "Unable to re-load f/w\n");
3855 err = mxge_open(sc);
3856 /* release all TX locks */
3857 for (s = 0; s < num_tx_slices; s++) {
3859 #ifdef IFNET_BUF_RING
3860 mxge_start_locked(ss);
3862 mtx_unlock(&ss->tx.mtx);
3865 sc->watchdog_resets++;
3867 device_printf(sc->dev,
3868 "NIC did not reboot, not resetting\n");
3872 device_printf(sc->dev, "watchdog reset failed\n");
3876 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
3881 mxge_watchdog_task(void *arg, int pending)
3883 mxge_softc_t *sc = arg;
3886 mtx_lock(&sc->driver_mtx);
3887 mxge_watchdog_reset(sc);
3888 mtx_unlock(&sc->driver_mtx);
3892 mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice)
3894 tx = &sc->ss[slice].tx;
3895 device_printf(sc->dev, "slice %d struck? ring state:\n", slice);
3896 device_printf(sc->dev,
3897 "tx.req=%d tx.done=%d, tx.queue_active=%d\n",
3898 tx->req, tx->done, tx->queue_active);
3899 device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n",
3900 tx->activate, tx->deactivate);
3901 device_printf(sc->dev, "pkt_done=%d fw=%d\n",
3903 be32toh(sc->ss->fw_stats->send_done_count));
3907 mxge_watchdog(mxge_softc_t *sc)
3910 uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause);
3913 /* see if we have outstanding transmits, which
3914 have been pending for more than mxge_ticks */
3916 #ifdef IFNET_BUF_RING
3917 (i < sc->num_slices) && (err == 0);
3919 (i < 1) && (err == 0);
3923 if (tx->req != tx->done &&
3924 tx->watchdog_req != tx->watchdog_done &&
3925 tx->done == tx->watchdog_done) {
3926 /* check for pause blocking before resetting */
3927 if (tx->watchdog_rx_pause == rx_pause) {
3928 mxge_warn_stuck(sc, tx, i);
3929 taskqueue_enqueue(sc->tq, &sc->watchdog_task);
3933 device_printf(sc->dev, "Flow control blocking "
3934 "xmits, check link partner\n");
3937 tx->watchdog_req = tx->req;
3938 tx->watchdog_done = tx->done;
3939 tx->watchdog_rx_pause = rx_pause;
3942 if (sc->need_media_probe)
3943 mxge_media_probe(sc);
3948 mxge_update_stats(mxge_softc_t *sc)
3950 struct mxge_slice_state *ss;
3952 u_long ipackets = 0;
3953 u_long opackets = 0;
3954 #ifdef IFNET_BUF_RING
3962 for (slice = 0; slice < sc->num_slices; slice++) {
3963 ss = &sc->ss[slice];
3964 ipackets += ss->ipackets;
3965 opackets += ss->opackets;
3966 #ifdef IFNET_BUF_RING
3967 obytes += ss->obytes;
3968 omcasts += ss->omcasts;
3969 odrops += ss->tx.br->br_drops;
3971 oerrors += ss->oerrors;
3973 pkts = (ipackets - sc->ifp->if_ipackets);
3974 pkts += (opackets - sc->ifp->if_opackets);
3975 sc->ifp->if_ipackets = ipackets;
3976 sc->ifp->if_opackets = opackets;
3977 #ifdef IFNET_BUF_RING
3978 sc->ifp->if_obytes = obytes;
3979 sc->ifp->if_omcasts = omcasts;
3980 sc->ifp->if_snd.ifq_drops = odrops;
3982 sc->ifp->if_oerrors = oerrors;
3987 mxge_tick(void *arg)
3989 mxge_softc_t *sc = arg;
3996 running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING;
3998 /* aggregate stats from different slices */
3999 pkts = mxge_update_stats(sc);
4000 if (!sc->watchdog_countdown) {
4001 err = mxge_watchdog(sc);
4002 sc->watchdog_countdown = 4;
4004 sc->watchdog_countdown--;
4007 /* ensure NIC did not suffer h/w fault while idle */
4008 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
4009 if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
4011 taskqueue_enqueue(sc->tq, &sc->watchdog_task);
4014 /* look less often if NIC is idle */
4019 callout_reset(&sc->co_hdl, ticks, mxge_tick, sc);
4024 mxge_media_change(struct ifnet *ifp)
4030 mxge_change_mtu(mxge_softc_t *sc, int mtu)
4032 struct ifnet *ifp = sc->ifp;
4033 int real_mtu, old_mtu;
4037 real_mtu = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
4038 if ((real_mtu > sc->max_mtu) || real_mtu < 60)
4040 mtx_lock(&sc->driver_mtx);
4041 old_mtu = ifp->if_mtu;
4043 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
4045 err = mxge_open(sc);
4047 ifp->if_mtu = old_mtu;
4049 (void) mxge_open(sc);
4052 mtx_unlock(&sc->driver_mtx);
4057 mxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
4059 mxge_softc_t *sc = ifp->if_softc;
4064 ifmr->ifm_status = IFM_AVALID;
4065 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
4066 ifmr->ifm_status |= sc->link_state ? IFM_ACTIVE : 0;
4067 ifmr->ifm_active |= sc->current_media;
4071 mxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
4073 mxge_softc_t *sc = ifp->if_softc;
4074 struct ifreq *ifr = (struct ifreq *)data;
4081 err = ether_ioctl(ifp, command, data);
4085 err = mxge_change_mtu(sc, ifr->ifr_mtu);
4089 mtx_lock(&sc->driver_mtx);
4091 mtx_unlock(&sc->driver_mtx);
4094 if (ifp->if_flags & IFF_UP) {
4095 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4096 err = mxge_open(sc);
4098 /* take care of promis can allmulti
4100 mxge_change_promisc(sc,
4101 ifp->if_flags & IFF_PROMISC);
4102 mxge_set_multicast_list(sc);
4105 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
4109 mtx_unlock(&sc->driver_mtx);
4114 mtx_lock(&sc->driver_mtx);
4115 mxge_set_multicast_list(sc);
4116 mtx_unlock(&sc->driver_mtx);
4120 mtx_lock(&sc->driver_mtx);
4121 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
4122 if (mask & IFCAP_TXCSUM) {
4123 if (IFCAP_TXCSUM & ifp->if_capenable) {
4124 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
4125 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
4128 ifp->if_capenable |= IFCAP_TXCSUM;
4129 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
4131 } else if (mask & IFCAP_RXCSUM) {
4132 if (IFCAP_RXCSUM & ifp->if_capenable) {
4133 ifp->if_capenable &= ~IFCAP_RXCSUM;
4136 ifp->if_capenable |= IFCAP_RXCSUM;
4140 if (mask & IFCAP_TSO4) {
4141 if (IFCAP_TSO4 & ifp->if_capenable) {
4142 ifp->if_capenable &= ~IFCAP_TSO4;
4143 ifp->if_hwassist &= ~CSUM_TSO;
4144 } else if (IFCAP_TXCSUM & ifp->if_capenable) {
4145 ifp->if_capenable |= IFCAP_TSO4;
4146 ifp->if_hwassist |= CSUM_TSO;
4148 printf("mxge requires tx checksum offload"
4149 " be enabled to use TSO\n");
4153 if (mask & IFCAP_LRO) {
4154 if (IFCAP_LRO & ifp->if_capenable)
4155 err = mxge_change_lro_locked(sc, 0);
4157 err = mxge_change_lro_locked(sc, mxge_lro_cnt);
4159 if (mask & IFCAP_VLAN_HWTAGGING)
4160 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
4161 if (mask & IFCAP_VLAN_HWTSO)
4162 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
4164 if (!(ifp->if_capabilities & IFCAP_VLAN_HWTSO) ||
4165 !(ifp->if_capenable & IFCAP_VLAN_HWTAGGING))
4166 ifp->if_capenable &= ~IFCAP_VLAN_HWTSO;
4168 mtx_unlock(&sc->driver_mtx);
4169 VLAN_CAPABILITIES(ifp);
4174 mtx_lock(&sc->driver_mtx);
4175 mxge_media_probe(sc);
4176 mtx_unlock(&sc->driver_mtx);
4177 err = ifmedia_ioctl(ifp, (struct ifreq *)data,
4178 &sc->media, command);
4189 mxge_fetch_tunables(mxge_softc_t *sc)
4192 TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices);
4193 TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled",
4194 &mxge_flow_control);
4195 TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay",
4196 &mxge_intr_coal_delay);
4197 TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable",
4198 &mxge_nvidia_ecrc_enable);
4199 TUNABLE_INT_FETCH("hw.mxge.force_firmware",
4200 &mxge_force_firmware);
4201 TUNABLE_INT_FETCH("hw.mxge.deassert_wait",
4202 &mxge_deassert_wait);
4203 TUNABLE_INT_FETCH("hw.mxge.verbose",
4205 TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks);
4206 TUNABLE_INT_FETCH("hw.mxge.lro_cnt", &sc->lro_cnt);
4207 TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc);
4208 TUNABLE_INT_FETCH("hw.mxge.rss_hash_type", &mxge_rss_hash_type);
4209 TUNABLE_INT_FETCH("hw.mxge.rss_hashtype", &mxge_rss_hash_type);
4210 TUNABLE_INT_FETCH("hw.mxge.initial_mtu", &mxge_initial_mtu);
4211 TUNABLE_INT_FETCH("hw.mxge.throttle", &mxge_throttle);
4212 if (sc->lro_cnt != 0)
4213 mxge_lro_cnt = sc->lro_cnt;
4217 if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000)
4218 mxge_intr_coal_delay = 30;
4219 if (mxge_ticks == 0)
4220 mxge_ticks = hz / 2;
4221 sc->pause = mxge_flow_control;
4222 if (mxge_rss_hash_type < MXGEFW_RSS_HASH_TYPE_IPV4
4223 || mxge_rss_hash_type > MXGEFW_RSS_HASH_TYPE_MAX) {
4224 mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
4226 if (mxge_initial_mtu > ETHERMTU_JUMBO ||
4227 mxge_initial_mtu < ETHER_MIN_LEN)
4228 mxge_initial_mtu = ETHERMTU_JUMBO;
4230 if (mxge_throttle && mxge_throttle > MXGE_MAX_THROTTLE)
4231 mxge_throttle = MXGE_MAX_THROTTLE;
4232 if (mxge_throttle && mxge_throttle < MXGE_MIN_THROTTLE)
4233 mxge_throttle = MXGE_MIN_THROTTLE;
4234 sc->throttle = mxge_throttle;
4239 mxge_free_slices(mxge_softc_t *sc)
4241 struct mxge_slice_state *ss;
4248 for (i = 0; i < sc->num_slices; i++) {
4250 if (ss->fw_stats != NULL) {
4251 mxge_dma_free(&ss->fw_stats_dma);
4252 ss->fw_stats = NULL;
4253 #ifdef IFNET_BUF_RING
4254 if (ss->tx.br != NULL) {
4255 drbr_free(ss->tx.br, M_DEVBUF);
4259 mtx_destroy(&ss->tx.mtx);
4261 if (ss->rx_done.entry != NULL) {
4262 mxge_dma_free(&ss->rx_done.dma);
4263 ss->rx_done.entry = NULL;
4266 free(sc->ss, M_DEVBUF);
4271 mxge_alloc_slices(mxge_softc_t *sc)
4274 struct mxge_slice_state *ss;
4276 int err, i, max_intr_slots;
4278 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
4280 device_printf(sc->dev, "Cannot determine rx ring size\n");
4283 sc->rx_ring_size = cmd.data0;
4284 max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t));
4286 bytes = sizeof (*sc->ss) * sc->num_slices;
4287 sc->ss = malloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO);
4290 for (i = 0; i < sc->num_slices; i++) {
4295 /* allocate per-slice rx interrupt queues */
4297 bytes = max_intr_slots * sizeof (*ss->rx_done.entry);
4298 err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096);
4301 ss->rx_done.entry = ss->rx_done.dma.addr;
4302 bzero(ss->rx_done.entry, bytes);
4305 * allocate the per-slice firmware stats; stats
4306 * (including tx) are used used only on the first
4309 #ifndef IFNET_BUF_RING
4314 bytes = sizeof (*ss->fw_stats);
4315 err = mxge_dma_alloc(sc, &ss->fw_stats_dma,
4316 sizeof (*ss->fw_stats), 64);
4319 ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr;
4320 snprintf(ss->tx.mtx_name, sizeof(ss->tx.mtx_name),
4321 "%s:tx(%d)", device_get_nameunit(sc->dev), i);
4322 mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF);
4323 #ifdef IFNET_BUF_RING
4324 ss->tx.br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK,
4332 mxge_free_slices(sc);
4337 mxge_slice_probe(mxge_softc_t *sc)
4341 int msix_cnt, status, max_intr_slots;
4345 * don't enable multiple slices if they are not enabled,
4346 * or if this is not an SMP system
4349 if (mxge_max_slices == 0 || mxge_max_slices == 1 || mp_ncpus < 2)
4352 /* see how many MSI-X interrupts are available */
4353 msix_cnt = pci_msix_count(sc->dev);
4357 /* now load the slice aware firmware see what it supports */
4358 old_fw = sc->fw_name;
4359 if (old_fw == mxge_fw_aligned)
4360 sc->fw_name = mxge_fw_rss_aligned;
4362 sc->fw_name = mxge_fw_rss_unaligned;
4363 status = mxge_load_firmware(sc, 0);
4365 device_printf(sc->dev, "Falling back to a single slice\n");
4369 /* try to send a reset command to the card to see if it
4371 memset(&cmd, 0, sizeof (cmd));
4372 status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
4374 device_printf(sc->dev, "failed reset\n");
4378 /* get rx ring size */
4379 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
4381 device_printf(sc->dev, "Cannot determine rx ring size\n");
4384 max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t));
4386 /* tell it the size of the interrupt queues */
4387 cmd.data0 = max_intr_slots * sizeof (struct mcp_slot);
4388 status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
4390 device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n");
4394 /* ask the maximum number of slices it supports */
4395 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd);
4397 device_printf(sc->dev,
4398 "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n");
4401 sc->num_slices = cmd.data0;
4402 if (sc->num_slices > msix_cnt)
4403 sc->num_slices = msix_cnt;
4405 if (mxge_max_slices == -1) {
4406 /* cap to number of CPUs in system */
4407 if (sc->num_slices > mp_ncpus)
4408 sc->num_slices = mp_ncpus;
4410 if (sc->num_slices > mxge_max_slices)
4411 sc->num_slices = mxge_max_slices;
4413 /* make sure it is a power of two */
4414 while (sc->num_slices & (sc->num_slices - 1))
4418 device_printf(sc->dev, "using %d slices\n",
4424 sc->fw_name = old_fw;
4425 (void) mxge_load_firmware(sc, 0);
4429 mxge_add_msix_irqs(mxge_softc_t *sc)
4432 int count, err, i, rid;
4435 sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
4438 if (sc->msix_table_res == NULL) {
4439 device_printf(sc->dev, "couldn't alloc MSIX table res\n");
4443 count = sc->num_slices;
4444 err = pci_alloc_msix(sc->dev, &count);
4446 device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d"
4447 "err = %d \n", sc->num_slices, err);
4448 goto abort_with_msix_table;
4450 if (count < sc->num_slices) {
4451 device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n",
4452 count, sc->num_slices);
4453 device_printf(sc->dev,
4454 "Try setting hw.mxge.max_slices to %d\n",
4457 goto abort_with_msix;
4459 bytes = sizeof (*sc->msix_irq_res) * sc->num_slices;
4460 sc->msix_irq_res = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
4461 if (sc->msix_irq_res == NULL) {
4463 goto abort_with_msix;
4466 for (i = 0; i < sc->num_slices; i++) {
4468 sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev,
4471 if (sc->msix_irq_res[i] == NULL) {
4472 device_printf(sc->dev, "couldn't allocate IRQ res"
4473 " for message %d\n", i);
4475 goto abort_with_res;
4479 bytes = sizeof (*sc->msix_ih) * sc->num_slices;
4480 sc->msix_ih = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
4482 for (i = 0; i < sc->num_slices; i++) {
4483 err = bus_setup_intr(sc->dev, sc->msix_irq_res[i],
4484 INTR_TYPE_NET | INTR_MPSAFE,
4485 #if __FreeBSD_version > 700030
4488 mxge_intr, &sc->ss[i], &sc->msix_ih[i]);
4490 device_printf(sc->dev, "couldn't setup intr for "
4492 goto abort_with_intr;
4494 bus_describe_intr(sc->dev, sc->msix_irq_res[i],
4495 sc->msix_ih[i], "s%d", i);
4499 device_printf(sc->dev, "using %d msix IRQs:",
4501 for (i = 0; i < sc->num_slices; i++)
4502 printf(" %ld", rman_get_start(sc->msix_irq_res[i]));
4508 for (i = 0; i < sc->num_slices; i++) {
4509 if (sc->msix_ih[i] != NULL) {
4510 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
4512 sc->msix_ih[i] = NULL;
4515 free(sc->msix_ih, M_DEVBUF);
4519 for (i = 0; i < sc->num_slices; i++) {
4521 if (sc->msix_irq_res[i] != NULL)
4522 bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
4523 sc->msix_irq_res[i]);
4524 sc->msix_irq_res[i] = NULL;
4526 free(sc->msix_irq_res, M_DEVBUF);
4530 pci_release_msi(sc->dev);
4532 abort_with_msix_table:
4533 bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
4534 sc->msix_table_res);
4540 mxge_add_single_irq(mxge_softc_t *sc)
4542 int count, err, rid;
4544 count = pci_msi_count(sc->dev);
4545 if (count == 1 && pci_alloc_msi(sc->dev, &count) == 0) {
4551 sc->irq_res = bus_alloc_resource(sc->dev, SYS_RES_IRQ, &rid, 0, ~0,
4552 1, RF_SHAREABLE | RF_ACTIVE);
4553 if (sc->irq_res == NULL) {
4554 device_printf(sc->dev, "could not alloc interrupt\n");
4558 device_printf(sc->dev, "using %s irq %ld\n",
4559 sc->legacy_irq ? "INTx" : "MSI",
4560 rman_get_start(sc->irq_res));
4561 err = bus_setup_intr(sc->dev, sc->irq_res,
4562 INTR_TYPE_NET | INTR_MPSAFE,
4563 #if __FreeBSD_version > 700030
4566 mxge_intr, &sc->ss[0], &sc->ih);
4568 bus_release_resource(sc->dev, SYS_RES_IRQ,
4569 sc->legacy_irq ? 0 : 1, sc->irq_res);
4570 if (!sc->legacy_irq)
4571 pci_release_msi(sc->dev);
4577 mxge_rem_msix_irqs(mxge_softc_t *sc)
4581 for (i = 0; i < sc->num_slices; i++) {
4582 if (sc->msix_ih[i] != NULL) {
4583 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
4585 sc->msix_ih[i] = NULL;
4588 free(sc->msix_ih, M_DEVBUF);
4590 for (i = 0; i < sc->num_slices; i++) {
4592 if (sc->msix_irq_res[i] != NULL)
4593 bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
4594 sc->msix_irq_res[i]);
4595 sc->msix_irq_res[i] = NULL;
4597 free(sc->msix_irq_res, M_DEVBUF);
4599 bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
4600 sc->msix_table_res);
4602 pci_release_msi(sc->dev);
4607 mxge_rem_single_irq(mxge_softc_t *sc)
4609 bus_teardown_intr(sc->dev, sc->irq_res, sc->ih);
4610 bus_release_resource(sc->dev, SYS_RES_IRQ,
4611 sc->legacy_irq ? 0 : 1, sc->irq_res);
4612 if (!sc->legacy_irq)
4613 pci_release_msi(sc->dev);
4617 mxge_rem_irq(mxge_softc_t *sc)
4619 if (sc->num_slices > 1)
4620 mxge_rem_msix_irqs(sc);
4622 mxge_rem_single_irq(sc);
4626 mxge_add_irq(mxge_softc_t *sc)
4630 if (sc->num_slices > 1)
4631 err = mxge_add_msix_irqs(sc);
4633 err = mxge_add_single_irq(sc);
4635 if (0 && err == 0 && sc->num_slices > 1) {
4636 mxge_rem_msix_irqs(sc);
4637 err = mxge_add_msix_irqs(sc);
4644 mxge_attach(device_t dev)
4646 mxge_softc_t *sc = device_get_softc(dev);
4651 mxge_fetch_tunables(sc);
4653 TASK_INIT(&sc->watchdog_task, 1, mxge_watchdog_task, sc);
4654 sc->tq = taskqueue_create_fast("mxge_taskq", M_WAITOK,
4655 taskqueue_thread_enqueue,
4657 if (sc->tq == NULL) {
4659 goto abort_with_nothing;
4662 err = bus_dma_tag_create(NULL, /* parent */
4665 BUS_SPACE_MAXADDR, /* low */
4666 BUS_SPACE_MAXADDR, /* high */
4667 NULL, NULL, /* filter */
4668 65536 + 256, /* maxsize */
4669 MXGE_MAX_SEND_DESC, /* num segs */
4670 65536, /* maxsegsize */
4672 NULL, NULL, /* lock */
4673 &sc->parent_dmat); /* tag */
4676 device_printf(sc->dev, "Err %d allocating parent dmat\n",
4681 ifp = sc->ifp = if_alloc(IFT_ETHER);
4683 device_printf(dev, "can not if_alloc()\n");
4685 goto abort_with_parent_dmat;
4687 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
4689 snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd",
4690 device_get_nameunit(dev));
4691 mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF);
4692 snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name),
4693 "%s:drv", device_get_nameunit(dev));
4694 mtx_init(&sc->driver_mtx, sc->driver_mtx_name,
4695 MTX_NETWORK_LOCK, MTX_DEF);
4697 callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0);
4699 mxge_setup_cfg_space(sc);
4701 /* Map the board into the kernel */
4703 sc->mem_res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 0,
4705 if (sc->mem_res == NULL) {
4706 device_printf(dev, "could not map memory\n");
4708 goto abort_with_lock;
4710 sc->sram = rman_get_virtual(sc->mem_res);
4711 sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100;
4712 if (sc->sram_size > rman_get_size(sc->mem_res)) {
4713 device_printf(dev, "impossible memory region size %ld\n",
4714 rman_get_size(sc->mem_res));
4716 goto abort_with_mem_res;
4719 /* make NULL terminated copy of the EEPROM strings section of
4721 bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE);
4722 bus_space_read_region_1(rman_get_bustag(sc->mem_res),
4723 rman_get_bushandle(sc->mem_res),
4724 sc->sram_size - MXGE_EEPROM_STRINGS_SIZE,
4726 MXGE_EEPROM_STRINGS_SIZE - 2);
4727 err = mxge_parse_strings(sc);
4729 goto abort_with_mem_res;
4731 /* Enable write combining for efficient use of PCIe bus */
4734 /* Allocate the out of band dma memory */
4735 err = mxge_dma_alloc(sc, &sc->cmd_dma,
4736 sizeof (mxge_cmd_t), 64);
4738 goto abort_with_mem_res;
4739 sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr;
4740 err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64);
4742 goto abort_with_cmd_dma;
4744 err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096);
4746 goto abort_with_zeropad_dma;
4748 /* select & load the firmware */
4749 err = mxge_select_firmware(sc);
4751 goto abort_with_dmabench;
4752 sc->intr_coal_delay = mxge_intr_coal_delay;
4754 mxge_slice_probe(sc);
4755 err = mxge_alloc_slices(sc);
4757 goto abort_with_dmabench;
4759 err = mxge_reset(sc, 0);
4761 goto abort_with_slices;
4763 err = mxge_alloc_rings(sc);
4765 device_printf(sc->dev, "failed to allocate rings\n");
4766 goto abort_with_slices;
4769 err = mxge_add_irq(sc);
4771 device_printf(sc->dev, "failed to add irq\n");
4772 goto abort_with_rings;
4775 ifp->if_baudrate = IF_Gbps(10UL);
4776 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 |
4779 ifp->if_capabilities |= IFCAP_LRO;
4782 #ifdef MXGE_NEW_VLAN_API
4783 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
4785 /* Only FW 1.4.32 and newer can do TSO over vlans */
4786 if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 &&
4787 sc->fw_ver_tiny >= 32)
4788 ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
4791 sc->max_mtu = mxge_max_mtu(sc);
4792 if (sc->max_mtu >= 9000)
4793 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
4795 device_printf(dev, "MTU limited to %d. Install "
4796 "latest firmware for 9000 byte jumbo support\n",
4797 sc->max_mtu - ETHER_HDR_LEN);
4798 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
4799 ifp->if_capenable = ifp->if_capabilities;
4800 if (sc->lro_cnt == 0)
4801 ifp->if_capenable &= ~IFCAP_LRO;
4803 ifp->if_init = mxge_init;
4805 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
4806 ifp->if_ioctl = mxge_ioctl;
4807 ifp->if_start = mxge_start;
4808 /* Initialise the ifmedia structure */
4809 ifmedia_init(&sc->media, 0, mxge_media_change,
4811 mxge_media_init(sc);
4812 mxge_media_probe(sc);
4814 ether_ifattach(ifp, sc->mac_addr);
4815 /* ether_ifattach sets mtu to ETHERMTU */
4816 if (mxge_initial_mtu != ETHERMTU)
4817 mxge_change_mtu(sc, mxge_initial_mtu);
4819 mxge_add_sysctls(sc);
4820 #ifdef IFNET_BUF_RING
4821 ifp->if_transmit = mxge_transmit;
4822 ifp->if_qflush = mxge_qflush;
4824 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
4825 device_get_nameunit(sc->dev));
4826 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
4830 mxge_free_rings(sc);
4832 mxge_free_slices(sc);
4833 abort_with_dmabench:
4834 mxge_dma_free(&sc->dmabench_dma);
4835 abort_with_zeropad_dma:
4836 mxge_dma_free(&sc->zeropad_dma);
4838 mxge_dma_free(&sc->cmd_dma);
4840 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
4842 pci_disable_busmaster(dev);
4843 mtx_destroy(&sc->cmd_mtx);
4844 mtx_destroy(&sc->driver_mtx);
4846 abort_with_parent_dmat:
4847 bus_dma_tag_destroy(sc->parent_dmat);
4849 if (sc->tq != NULL) {
4850 taskqueue_drain(sc->tq, &sc->watchdog_task);
4851 taskqueue_free(sc->tq);
4859 mxge_detach(device_t dev)
4861 mxge_softc_t *sc = device_get_softc(dev);
4863 if (mxge_vlans_active(sc)) {
4864 device_printf(sc->dev,
4865 "Detach vlans before removing module\n");
4868 mtx_lock(&sc->driver_mtx);
4870 if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING)
4872 mtx_unlock(&sc->driver_mtx);
4873 ether_ifdetach(sc->ifp);
4874 if (sc->tq != NULL) {
4875 taskqueue_drain(sc->tq, &sc->watchdog_task);
4876 taskqueue_free(sc->tq);
4879 callout_drain(&sc->co_hdl);
4880 ifmedia_removeall(&sc->media);
4881 mxge_dummy_rdma(sc, 0);
4882 mxge_rem_sysctls(sc);
4884 mxge_free_rings(sc);
4885 mxge_free_slices(sc);
4886 mxge_dma_free(&sc->dmabench_dma);
4887 mxge_dma_free(&sc->zeropad_dma);
4888 mxge_dma_free(&sc->cmd_dma);
4889 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
4890 pci_disable_busmaster(dev);
4891 mtx_destroy(&sc->cmd_mtx);
4892 mtx_destroy(&sc->driver_mtx);
4894 bus_dma_tag_destroy(sc->parent_dmat);
4899 mxge_shutdown(device_t dev)
4905 This file uses Myri10GE driver indentation.
4908 c-file-style:"linux"