1 /******************************************************************************
3 Copyright (c) 2001-2013, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
39 #ifdef HAVE_KERNEL_OPTION_HEADERS
40 #include "opt_device_polling.h"
44 #include <sys/param.h>
45 #include <sys/systm.h>
47 #include <sys/buf_ring.h>
50 #include <sys/endian.h>
51 #include <sys/kernel.h>
52 #include <sys/kthread.h>
53 #include <sys/malloc.h>
55 #include <sys/module.h>
57 #include <sys/socket.h>
58 #include <sys/sockio.h>
59 #include <sys/sysctl.h>
60 #include <sys/taskqueue.h>
61 #include <sys/eventhandler.h>
64 #include <machine/smp.h>
65 #include <machine/bus.h>
66 #include <machine/resource.h>
69 #include <net/ethernet.h>
71 #include <net/if_arp.h>
72 #include <net/if_dl.h>
73 #include <net/if_media.h>
75 #include <net/if_types.h>
76 #include <net/if_vlan_var.h>
78 #include <netinet/in_systm.h>
79 #include <netinet/in.h>
80 #include <netinet/if_ether.h>
81 #include <netinet/ip.h>
82 #include <netinet/ip6.h>
83 #include <netinet/tcp.h>
84 #include <netinet/tcp_lro.h>
85 #include <netinet/udp.h>
87 #include <machine/in_cksum.h>
88 #include <dev/led/led.h>
89 #include <dev/pci/pcivar.h>
90 #include <dev/pci/pcireg.h>
92 #include "e1000_api.h"
93 #include "e1000_82575.h"
96 /*********************************************************************
97 * Set this to one to display debug statistics
98 *********************************************************************/
99 int igb_display_debug_stats = 0;
101 /*********************************************************************
103 *********************************************************************/
104 char igb_driver_version[] = "version - 2.4.0";
107 /*********************************************************************
108 * PCI Device ID Table
110 * Used by probe to select devices to load on
111 * Last field stores an index into e1000_strings
112 * Last entry must be all 0s
114 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
115 *********************************************************************/
117 static igb_vendor_info_t igb_vendor_info_array[] =
119 { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
120 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
121 PCI_ANY_ID, PCI_ANY_ID, 0},
122 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
123 PCI_ANY_ID, PCI_ANY_ID, 0},
124 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0},
126 { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
128 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
129 { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
130 PCI_ANY_ID, PCI_ANY_ID, 0},
131 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
132 PCI_ANY_ID, PCI_ANY_ID, 0},
133 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2,
134 PCI_ANY_ID, PCI_ANY_ID, 0},
135 { 0x8086, E1000_DEV_ID_82576_VF, PCI_ANY_ID, PCI_ANY_ID, 0},
136 { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
137 { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
138 { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
139 { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
140 { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL,
141 PCI_ANY_ID, PCI_ANY_ID, 0},
142 { 0x8086, E1000_DEV_ID_82580_QUAD_FIBER,
143 PCI_ANY_ID, PCI_ANY_ID, 0},
144 { 0x8086, E1000_DEV_ID_DH89XXCC_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
145 { 0x8086, E1000_DEV_ID_DH89XXCC_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_DH89XXCC_SFP, PCI_ANY_ID, PCI_ANY_ID, 0},
147 { 0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE,
148 PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_I350_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_I350_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
151 { 0x8086, E1000_DEV_ID_I350_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
152 { 0x8086, E1000_DEV_ID_I350_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
153 { 0x8086, E1000_DEV_ID_I350_VF, PCI_ANY_ID, PCI_ANY_ID, 0},
154 { 0x8086, E1000_DEV_ID_I210_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
155 { 0x8086, E1000_DEV_ID_I210_COPPER_IT, PCI_ANY_ID, PCI_ANY_ID, 0},
156 { 0x8086, E1000_DEV_ID_I210_COPPER_OEM1,
157 PCI_ANY_ID, PCI_ANY_ID, 0},
158 { 0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS,
159 PCI_ANY_ID, PCI_ANY_ID, 0},
160 { 0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS,
161 PCI_ANY_ID, PCI_ANY_ID, 0},
162 { 0x8086, E1000_DEV_ID_I210_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
163 { 0x8086, E1000_DEV_ID_I210_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
164 { 0x8086, E1000_DEV_ID_I210_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
165 { 0x8086, E1000_DEV_ID_I211_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
166 { 0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS,
167 PCI_ANY_ID, PCI_ANY_ID, 0},
168 { 0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS,
169 PCI_ANY_ID, PCI_ANY_ID, 0},
170 { 0x8086, E1000_DEV_ID_I354_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
171 /* required last entry */
175 /*********************************************************************
176 * Table of branding strings for all supported NICs.
177 *********************************************************************/
179 static char *igb_strings[] = {
180 "Intel(R) PRO/1000 Network Connection"
183 /*********************************************************************
184 * Function prototypes
185 *********************************************************************/
186 static int igb_probe(device_t);
187 static int igb_attach(device_t);
188 static int igb_detach(device_t);
189 static int igb_shutdown(device_t);
190 static int igb_suspend(device_t);
191 static int igb_resume(device_t);
192 #ifndef IGB_LEGACY_TX
193 static int igb_mq_start(struct ifnet *, struct mbuf *);
194 static int igb_mq_start_locked(struct ifnet *, struct tx_ring *);
195 static void igb_qflush(struct ifnet *);
196 static void igb_deferred_mq_start(void *, int);
198 static void igb_start(struct ifnet *);
199 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
201 static int igb_ioctl(struct ifnet *, u_long, caddr_t);
202 static void igb_init(void *);
203 static void igb_init_locked(struct adapter *);
204 static void igb_stop(void *);
205 static void igb_media_status(struct ifnet *, struct ifmediareq *);
206 static int igb_media_change(struct ifnet *);
207 static void igb_identify_hardware(struct adapter *);
208 static int igb_allocate_pci_resources(struct adapter *);
209 static int igb_allocate_msix(struct adapter *);
210 static int igb_allocate_legacy(struct adapter *);
211 static int igb_setup_msix(struct adapter *);
212 static void igb_free_pci_resources(struct adapter *);
213 static void igb_local_timer(void *);
214 static void igb_reset(struct adapter *);
215 static int igb_setup_interface(device_t, struct adapter *);
216 static int igb_allocate_queues(struct adapter *);
217 static void igb_configure_queues(struct adapter *);
219 static int igb_allocate_transmit_buffers(struct tx_ring *);
220 static void igb_setup_transmit_structures(struct adapter *);
221 static void igb_setup_transmit_ring(struct tx_ring *);
222 static void igb_initialize_transmit_units(struct adapter *);
223 static void igb_free_transmit_structures(struct adapter *);
224 static void igb_free_transmit_buffers(struct tx_ring *);
226 static int igb_allocate_receive_buffers(struct rx_ring *);
227 static int igb_setup_receive_structures(struct adapter *);
228 static int igb_setup_receive_ring(struct rx_ring *);
229 static void igb_initialize_receive_units(struct adapter *);
230 static void igb_free_receive_structures(struct adapter *);
231 static void igb_free_receive_buffers(struct rx_ring *);
232 static void igb_free_receive_ring(struct rx_ring *);
234 static void igb_enable_intr(struct adapter *);
235 static void igb_disable_intr(struct adapter *);
236 static void igb_update_stats_counters(struct adapter *);
237 static bool igb_txeof(struct tx_ring *);
239 static __inline void igb_rx_discard(struct rx_ring *, int);
240 static __inline void igb_rx_input(struct rx_ring *,
241 struct ifnet *, struct mbuf *, u32);
243 static bool igb_rxeof(struct igb_queue *, int, int *);
244 static void igb_rx_checksum(u32, struct mbuf *, u32);
245 static int igb_tx_ctx_setup(struct tx_ring *,
246 struct mbuf *, u32 *, u32 *);
247 static int igb_tso_setup(struct tx_ring *,
248 struct mbuf *, u32 *, u32 *);
249 static void igb_set_promisc(struct adapter *);
250 static void igb_disable_promisc(struct adapter *);
251 static void igb_set_multi(struct adapter *);
252 static void igb_update_link_status(struct adapter *);
253 static void igb_refresh_mbufs(struct rx_ring *, int);
255 static void igb_register_vlan(void *, struct ifnet *, u16);
256 static void igb_unregister_vlan(void *, struct ifnet *, u16);
257 static void igb_setup_vlan_hw_support(struct adapter *);
259 static int igb_xmit(struct tx_ring *, struct mbuf **);
260 static int igb_dma_malloc(struct adapter *, bus_size_t,
261 struct igb_dma_alloc *, int);
262 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
263 static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
264 static void igb_print_nvm_info(struct adapter *);
265 static int igb_is_valid_ether_addr(u8 *);
266 static void igb_add_hw_stats(struct adapter *);
268 static void igb_vf_init_stats(struct adapter *);
269 static void igb_update_vf_stats_counters(struct adapter *);
271 /* Management and WOL Support */
272 static void igb_init_manageability(struct adapter *);
273 static void igb_release_manageability(struct adapter *);
274 static void igb_get_hw_control(struct adapter *);
275 static void igb_release_hw_control(struct adapter *);
276 static void igb_enable_wakeup(device_t);
277 static void igb_led_func(void *, int);
279 static int igb_irq_fast(void *);
280 static void igb_msix_que(void *);
281 static void igb_msix_link(void *);
282 static void igb_handle_que(void *context, int pending);
283 static void igb_handle_link(void *context, int pending);
284 static void igb_handle_link_locked(struct adapter *);
286 static void igb_set_sysctl_value(struct adapter *, const char *,
287 const char *, int *, int);
288 static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
289 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
290 static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
292 #ifdef DEVICE_POLLING
293 static poll_handler_t igb_poll;
296 /*********************************************************************
297 * FreeBSD Device Interface Entry Points
298 *********************************************************************/
300 static device_method_t igb_methods[] = {
301 /* Device interface */
302 DEVMETHOD(device_probe, igb_probe),
303 DEVMETHOD(device_attach, igb_attach),
304 DEVMETHOD(device_detach, igb_detach),
305 DEVMETHOD(device_shutdown, igb_shutdown),
306 DEVMETHOD(device_suspend, igb_suspend),
307 DEVMETHOD(device_resume, igb_resume),
311 static driver_t igb_driver = {
312 "igb", igb_methods, sizeof(struct adapter),
315 static devclass_t igb_devclass;
316 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
317 MODULE_DEPEND(igb, pci, 1, 1, 1);
318 MODULE_DEPEND(igb, ether, 1, 1, 1);
320 /*********************************************************************
321 * Tunable default values.
322 *********************************************************************/
324 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
326 /* Descriptor defaults */
327 static int igb_rxd = IGB_DEFAULT_RXD;
328 static int igb_txd = IGB_DEFAULT_TXD;
329 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
330 TUNABLE_INT("hw.igb.txd", &igb_txd);
331 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
332 "Number of receive descriptors per queue");
333 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
334 "Number of transmit descriptors per queue");
337 ** AIM: Adaptive Interrupt Moderation
338 ** which means that the interrupt rate
339 ** is varied over time based on the
340 ** traffic for that interrupt vector
342 static int igb_enable_aim = TRUE;
343 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
344 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0,
345 "Enable adaptive interrupt moderation");
348 * MSIX should be the default for best performance,
349 * but this allows it to be forced off for testing.
351 static int igb_enable_msix = 1;
352 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
353 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
354 "Enable MSI-X interrupts");
357 ** Tuneable Interrupt rate
359 static int igb_max_interrupt_rate = 8000;
360 TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate);
361 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
362 &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
364 #ifndef IGB_LEGACY_TX
366 ** Tuneable number of buffers in the buf-ring (drbr_xxx)
368 static int igb_buf_ring_size = IGB_BR_SIZE;
369 TUNABLE_INT("hw.igb.buf_ring_size", &igb_buf_ring_size);
370 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
371 &igb_buf_ring_size, 0, "Size of the bufring");
375 ** Header split causes the packet header to
376 ** be dma'd to a seperate mbuf from the payload.
377 ** this can have memory alignment benefits. But
378 ** another plus is that small packets often fit
379 ** into the header and thus use no cluster. Its
380 ** a very workload dependent type feature.
382 static int igb_header_split = FALSE;
383 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
384 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
385 "Enable receive mbuf header split");
388 ** This will autoconfigure based on the
389 ** number of CPUs and max supported
390 ** MSIX messages if left at 0.
392 static int igb_num_queues = 0;
393 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
394 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
395 "Number of queues to configure, 0 indicates autoconfigure");
398 ** Global variable to store last used CPU when binding queues
399 ** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
400 ** queue is bound to a cpu.
402 static int igb_last_bind_cpu = -1;
404 /* How many packets rxeof tries to clean at a time */
405 static int igb_rx_process_limit = 100;
406 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
407 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
408 &igb_rx_process_limit, 0,
409 "Maximum number of received packets to process at a time, -1 means unlimited");
411 #ifdef DEV_NETMAP /* see ixgbe.c for details */
412 #include <dev/netmap/if_igb_netmap.h>
413 #endif /* DEV_NETMAP */
414 /*********************************************************************
415 * Device identification routine
417 * igb_probe determines if the driver should be loaded on
418 * adapter based on PCI vendor/device id of the adapter.
420 * return BUS_PROBE_DEFAULT on success, positive on failure
421 *********************************************************************/
424 igb_probe(device_t dev)
426 char adapter_name[60];
427 uint16_t pci_vendor_id = 0;
428 uint16_t pci_device_id = 0;
429 uint16_t pci_subvendor_id = 0;
430 uint16_t pci_subdevice_id = 0;
431 igb_vendor_info_t *ent;
433 INIT_DEBUGOUT("igb_probe: begin");
435 pci_vendor_id = pci_get_vendor(dev);
436 if (pci_vendor_id != IGB_VENDOR_ID)
439 pci_device_id = pci_get_device(dev);
440 pci_subvendor_id = pci_get_subvendor(dev);
441 pci_subdevice_id = pci_get_subdevice(dev);
443 ent = igb_vendor_info_array;
444 while (ent->vendor_id != 0) {
445 if ((pci_vendor_id == ent->vendor_id) &&
446 (pci_device_id == ent->device_id) &&
448 ((pci_subvendor_id == ent->subvendor_id) ||
449 (ent->subvendor_id == PCI_ANY_ID)) &&
451 ((pci_subdevice_id == ent->subdevice_id) ||
452 (ent->subdevice_id == PCI_ANY_ID))) {
453 sprintf(adapter_name, "%s %s",
454 igb_strings[ent->index],
456 device_set_desc_copy(dev, adapter_name);
457 return (BUS_PROBE_DEFAULT);
465 /*********************************************************************
466 * Device initialization routine
468 * The attach entry point is called when the driver is being loaded.
469 * This routine identifies the type of hardware, allocates all resources
470 * and initializes the hardware.
472 * return 0 on success, positive on failure
473 *********************************************************************/
476 igb_attach(device_t dev)
478 struct adapter *adapter;
482 INIT_DEBUGOUT("igb_attach: begin");
484 if (resource_disabled("igb", device_get_unit(dev))) {
485 device_printf(dev, "Disabled by device hint\n");
489 adapter = device_get_softc(dev);
490 adapter->dev = adapter->osdep.dev = dev;
491 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
494 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
495 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
496 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
497 igb_sysctl_nvm_info, "I", "NVM Information");
499 igb_set_sysctl_value(adapter, "enable_aim",
500 "Interrupt Moderation", &adapter->enable_aim,
503 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
504 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
505 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
506 adapter, 0, igb_set_flowcntl, "I", "Flow Control");
508 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
510 /* Determine hardware and mac info */
511 igb_identify_hardware(adapter);
513 /* Setup PCI resources */
514 if (igb_allocate_pci_resources(adapter)) {
515 device_printf(dev, "Allocation of PCI resources failed\n");
520 /* Do Shared Code initialization */
521 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
522 device_printf(dev, "Setup of Shared code failed\n");
527 e1000_get_bus_info(&adapter->hw);
529 /* Sysctl for limiting the amount of work done in the taskqueue */
530 igb_set_sysctl_value(adapter, "rx_processing_limit",
531 "max number of rx packets to process",
532 &adapter->rx_process_limit, igb_rx_process_limit);
535 * Validate number of transmit and receive descriptors. It
536 * must not exceed hardware maximum, and must be multiple
537 * of E1000_DBA_ALIGN.
539 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
540 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
541 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
542 IGB_DEFAULT_TXD, igb_txd);
543 adapter->num_tx_desc = IGB_DEFAULT_TXD;
545 adapter->num_tx_desc = igb_txd;
546 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
547 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
548 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
549 IGB_DEFAULT_RXD, igb_rxd);
550 adapter->num_rx_desc = IGB_DEFAULT_RXD;
552 adapter->num_rx_desc = igb_rxd;
554 adapter->hw.mac.autoneg = DO_AUTO_NEG;
555 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
556 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
559 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
560 adapter->hw.phy.mdix = AUTO_ALL_MODES;
561 adapter->hw.phy.disable_polarity_correction = FALSE;
562 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
566 * Set the frame limits assuming
567 * standard ethernet sized frames.
569 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
572 ** Allocate and Setup Queues
574 if (igb_allocate_queues(adapter)) {
579 /* Allocate the appropriate stats memory */
580 if (adapter->vf_ifp) {
582 (struct e1000_vf_stats *)malloc(sizeof \
583 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
584 igb_vf_init_stats(adapter);
587 (struct e1000_hw_stats *)malloc(sizeof \
588 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
589 if (adapter->stats == NULL) {
590 device_printf(dev, "Can not allocate stats memory\n");
595 /* Allocate multicast array memory. */
596 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
597 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
598 if (adapter->mta == NULL) {
599 device_printf(dev, "Can not allocate multicast setup array\n");
604 /* Some adapter-specific advanced features */
605 if (adapter->hw.mac.type >= e1000_i350) {
606 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
607 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
608 OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
609 adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
610 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
611 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
612 OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
613 adapter, 0, igb_sysctl_eee, "I",
614 "Disable Energy Efficient Ethernet");
615 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
616 if (adapter->hw.mac.type == e1000_i354)
617 e1000_set_eee_i354(&adapter->hw);
619 e1000_set_eee_i350(&adapter->hw);
624 ** Start from a known state, this is
625 ** important in reading the nvm and
628 e1000_reset_hw(&adapter->hw);
630 /* Make sure we have a good EEPROM before we read from it */
631 if (((adapter->hw.mac.type != e1000_i210) &&
632 (adapter->hw.mac.type != e1000_i211)) &&
633 (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
635 ** Some PCI-E parts fail the first check due to
636 ** the link being in sleep state, call it again,
637 ** if it fails a second time its a real issue.
639 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
641 "The EEPROM Checksum Is Not Valid\n");
648 ** Copy the permanent MAC address out of the EEPROM
650 if (e1000_read_mac_addr(&adapter->hw) < 0) {
651 device_printf(dev, "EEPROM read error while reading MAC"
656 /* Check its sanity */
657 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
658 device_printf(dev, "Invalid MAC address\n");
663 /* Setup OS specific network interface */
664 if (igb_setup_interface(dev, adapter) != 0)
667 /* Now get a good starting state */
670 /* Initialize statistics */
671 igb_update_stats_counters(adapter);
673 adapter->hw.mac.get_link_status = 1;
674 igb_update_link_status(adapter);
676 /* Indicate SOL/IDER usage */
677 if (e1000_check_reset_block(&adapter->hw))
679 "PHY reset is blocked due to SOL/IDER session.\n");
681 /* Determine if we have to control management hardware */
682 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
687 /* APME bit in EEPROM is mapped to WUC.APME */
688 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
690 adapter->wol = E1000_WUFC_MAG;
692 /* Register for VLAN events */
693 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
694 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
695 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
696 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
698 igb_add_hw_stats(adapter);
700 /* Tell the stack that the interface is not active */
701 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
702 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
704 adapter->led_dev = led_create(igb_led_func, adapter,
705 device_get_nameunit(dev));
708 ** Configure Interrupts
710 if ((adapter->msix > 1) && (igb_enable_msix))
711 error = igb_allocate_msix(adapter);
712 else /* MSI or Legacy */
713 error = igb_allocate_legacy(adapter);
718 igb_netmap_attach(adapter);
719 #endif /* DEV_NETMAP */
720 INIT_DEBUGOUT("igb_attach: end");
726 igb_free_transmit_structures(adapter);
727 igb_free_receive_structures(adapter);
728 igb_release_hw_control(adapter);
730 igb_free_pci_resources(adapter);
731 if (adapter->ifp != NULL)
732 if_free(adapter->ifp);
733 free(adapter->mta, M_DEVBUF);
734 IGB_CORE_LOCK_DESTROY(adapter);
739 /*********************************************************************
740 * Device removal routine
742 * The detach entry point is called when the driver is being removed.
743 * This routine stops the adapter and deallocates all the resources
744 * that were allocated for driver operation.
746 * return 0 on success, positive on failure
747 *********************************************************************/
750 igb_detach(device_t dev)
752 struct adapter *adapter = device_get_softc(dev);
753 struct ifnet *ifp = adapter->ifp;
755 INIT_DEBUGOUT("igb_detach: begin");
757 /* Make sure VLANS are not using driver */
758 if (adapter->ifp->if_vlantrunk != NULL) {
759 device_printf(dev,"Vlan in use, detach first\n");
763 ether_ifdetach(adapter->ifp);
765 if (adapter->led_dev != NULL)
766 led_destroy(adapter->led_dev);
768 #ifdef DEVICE_POLLING
769 if (ifp->if_capenable & IFCAP_POLLING)
770 ether_poll_deregister(ifp);
773 IGB_CORE_LOCK(adapter);
774 adapter->in_detach = 1;
776 IGB_CORE_UNLOCK(adapter);
778 e1000_phy_hw_reset(&adapter->hw);
780 /* Give control back to firmware */
781 igb_release_manageability(adapter);
782 igb_release_hw_control(adapter);
785 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
786 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
787 igb_enable_wakeup(dev);
790 /* Unregister VLAN events */
791 if (adapter->vlan_attach != NULL)
792 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
793 if (adapter->vlan_detach != NULL)
794 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
796 callout_drain(&adapter->timer);
799 netmap_detach(adapter->ifp);
800 #endif /* DEV_NETMAP */
801 igb_free_pci_resources(adapter);
802 bus_generic_detach(dev);
805 igb_free_transmit_structures(adapter);
806 igb_free_receive_structures(adapter);
807 if (adapter->mta != NULL)
808 free(adapter->mta, M_DEVBUF);
810 IGB_CORE_LOCK_DESTROY(adapter);
815 /*********************************************************************
817 * Shutdown entry point
819 **********************************************************************/
822 igb_shutdown(device_t dev)
824 return igb_suspend(dev);
828 * Suspend/resume device methods.
831 igb_suspend(device_t dev)
833 struct adapter *adapter = device_get_softc(dev);
835 IGB_CORE_LOCK(adapter);
839 igb_release_manageability(adapter);
840 igb_release_hw_control(adapter);
843 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
844 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
845 igb_enable_wakeup(dev);
848 IGB_CORE_UNLOCK(adapter);
850 return bus_generic_suspend(dev);
854 igb_resume(device_t dev)
856 struct adapter *adapter = device_get_softc(dev);
857 struct tx_ring *txr = adapter->tx_rings;
858 struct ifnet *ifp = adapter->ifp;
860 IGB_CORE_LOCK(adapter);
861 igb_init_locked(adapter);
862 igb_init_manageability(adapter);
864 if ((ifp->if_flags & IFF_UP) &&
865 (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
866 for (int i = 0; i < adapter->num_queues; i++, txr++) {
868 #ifndef IGB_LEGACY_TX
869 /* Process the stack queue only if not depleted */
870 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
871 !drbr_empty(ifp, txr->br))
872 igb_mq_start_locked(ifp, txr);
874 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
875 igb_start_locked(txr, ifp);
880 IGB_CORE_UNLOCK(adapter);
882 return bus_generic_resume(dev);
888 /*********************************************************************
889 * Transmit entry point
891 * igb_start is called by the stack to initiate a transmit.
892 * The driver will remain in this routine as long as there are
893 * packets to transmit and transmit resources are available.
894 * In case resources are not available stack is notified and
895 * the packet is requeued.
896 **********************************************************************/
899 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
901 struct adapter *adapter = ifp->if_softc;
904 IGB_TX_LOCK_ASSERT(txr);
906 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
909 if (!adapter->link_active)
912 /* Call cleanup if number of TX descriptors low */
913 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
916 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
917 if (txr->tx_avail <= IGB_MAX_SCATTER) {
918 txr->queue_status |= IGB_QUEUE_DEPLETED;
921 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
925 * Encapsulation can modify our pointer, and or make it
926 * NULL on failure. In that event, we can't requeue.
928 if (igb_xmit(txr, &m_head)) {
930 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
931 if (txr->tx_avail <= IGB_MAX_SCATTER)
932 txr->queue_status |= IGB_QUEUE_DEPLETED;
936 /* Send a copy of the frame to the BPF listener */
937 ETHER_BPF_MTAP(ifp, m_head);
939 /* Set watchdog on */
940 txr->watchdog_time = ticks;
941 txr->queue_status |= IGB_QUEUE_WORKING;
946 * Legacy TX driver routine, called from the
947 * stack, always uses tx[0], and spins for it.
948 * Should not be used with multiqueue tx
951 igb_start(struct ifnet *ifp)
953 struct adapter *adapter = ifp->if_softc;
954 struct tx_ring *txr = adapter->tx_rings;
956 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
958 igb_start_locked(txr, ifp);
964 #else /* ~IGB_LEGACY_TX */
967 ** Multiqueue Transmit Entry:
968 ** quick turnaround to the stack
972 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
974 struct adapter *adapter = ifp->if_softc;
975 struct igb_queue *que;
979 /* Which queue to use */
980 if ((m->m_flags & M_FLOWID) != 0)
981 i = m->m_pkthdr.flowid % adapter->num_queues;
983 i = curcpu % adapter->num_queues;
984 txr = &adapter->tx_rings[i];
985 que = &adapter->queues[i];
987 err = drbr_enqueue(ifp, txr->br, m);
990 if (IGB_TX_TRYLOCK(txr)) {
991 err = igb_mq_start_locked(ifp, txr);
994 taskqueue_enqueue(que->tq, &txr->txq_task);
1000 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
1002 struct adapter *adapter = txr->adapter;
1004 int err = 0, enq = 0;
1006 IGB_TX_LOCK_ASSERT(txr);
1008 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
1009 adapter->link_active == 0)
1013 /* Process the queue */
1014 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
1015 if ((err = igb_xmit(txr, &next)) != 0) {
1017 /* It was freed, move forward */
1018 drbr_advance(ifp, txr->br);
1021 * Still have one left, it may not be
1022 * the same since the transmit function
1023 * may have changed it.
1025 drbr_putback(ifp, txr->br, next);
1029 drbr_advance(ifp, txr->br);
1031 ifp->if_obytes += next->m_pkthdr.len;
1032 if (next->m_flags & M_MCAST)
1034 ETHER_BPF_MTAP(ifp, next);
1035 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1039 /* Set the watchdog */
1040 txr->queue_status |= IGB_QUEUE_WORKING;
1041 txr->watchdog_time = ticks;
1043 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
1045 if (txr->tx_avail <= IGB_MAX_SCATTER)
1046 txr->queue_status |= IGB_QUEUE_DEPLETED;
1051 * Called from a taskqueue to drain queued transmit packets.
1054 igb_deferred_mq_start(void *arg, int pending)
1056 struct tx_ring *txr = arg;
1057 struct adapter *adapter = txr->adapter;
1058 struct ifnet *ifp = adapter->ifp;
1061 if (!drbr_empty(ifp, txr->br))
1062 igb_mq_start_locked(ifp, txr);
1067 ** Flush all ring buffers
1070 igb_qflush(struct ifnet *ifp)
1072 struct adapter *adapter = ifp->if_softc;
1073 struct tx_ring *txr = adapter->tx_rings;
1076 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1078 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1084 #endif /* ~IGB_LEGACY_TX */
1086 /*********************************************************************
1089 * igb_ioctl is called when the user wants to configure the
1092 * return 0 on success, positive on failure
1093 **********************************************************************/
1096 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1098 struct adapter *adapter = ifp->if_softc;
1099 struct ifreq *ifr = (struct ifreq *)data;
1100 #if defined(INET) || defined(INET6)
1101 struct ifaddr *ifa = (struct ifaddr *)data;
1103 bool avoid_reset = FALSE;
1106 if (adapter->in_detach)
1112 if (ifa->ifa_addr->sa_family == AF_INET)
1116 if (ifa->ifa_addr->sa_family == AF_INET6)
1120 ** Calling init results in link renegotiation,
1121 ** so we avoid doing it when possible.
1124 ifp->if_flags |= IFF_UP;
1125 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1128 if (!(ifp->if_flags & IFF_NOARP))
1129 arp_ifinit(ifp, ifa);
1132 error = ether_ioctl(ifp, command, data);
1138 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1140 IGB_CORE_LOCK(adapter);
1141 max_frame_size = 9234;
1142 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1144 IGB_CORE_UNLOCK(adapter);
1149 ifp->if_mtu = ifr->ifr_mtu;
1150 adapter->max_frame_size =
1151 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1152 igb_init_locked(adapter);
1153 IGB_CORE_UNLOCK(adapter);
1157 IOCTL_DEBUGOUT("ioctl rcv'd:\
1158 SIOCSIFFLAGS (Set Interface Flags)");
1159 IGB_CORE_LOCK(adapter);
1160 if (ifp->if_flags & IFF_UP) {
1161 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1162 if ((ifp->if_flags ^ adapter->if_flags) &
1163 (IFF_PROMISC | IFF_ALLMULTI)) {
1164 igb_disable_promisc(adapter);
1165 igb_set_promisc(adapter);
1168 igb_init_locked(adapter);
1170 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1172 adapter->if_flags = ifp->if_flags;
1173 IGB_CORE_UNLOCK(adapter);
1177 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1178 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1179 IGB_CORE_LOCK(adapter);
1180 igb_disable_intr(adapter);
1181 igb_set_multi(adapter);
1182 #ifdef DEVICE_POLLING
1183 if (!(ifp->if_capenable & IFCAP_POLLING))
1185 igb_enable_intr(adapter);
1186 IGB_CORE_UNLOCK(adapter);
1190 /* Check SOL/IDER usage */
1191 IGB_CORE_LOCK(adapter);
1192 if (e1000_check_reset_block(&adapter->hw)) {
1193 IGB_CORE_UNLOCK(adapter);
1194 device_printf(adapter->dev, "Media change is"
1195 " blocked due to SOL/IDER session.\n");
1198 IGB_CORE_UNLOCK(adapter);
1200 IOCTL_DEBUGOUT("ioctl rcv'd: \
1201 SIOCxIFMEDIA (Get/Set Interface Media)");
1202 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1208 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1210 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1211 #ifdef DEVICE_POLLING
1212 if (mask & IFCAP_POLLING) {
1213 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1214 error = ether_poll_register(igb_poll, ifp);
1217 IGB_CORE_LOCK(adapter);
1218 igb_disable_intr(adapter);
1219 ifp->if_capenable |= IFCAP_POLLING;
1220 IGB_CORE_UNLOCK(adapter);
1222 error = ether_poll_deregister(ifp);
1223 /* Enable interrupt even in error case */
1224 IGB_CORE_LOCK(adapter);
1225 igb_enable_intr(adapter);
1226 ifp->if_capenable &= ~IFCAP_POLLING;
1227 IGB_CORE_UNLOCK(adapter);
1231 if (mask & IFCAP_HWCSUM) {
1232 ifp->if_capenable ^= IFCAP_HWCSUM;
1235 if (mask & IFCAP_TSO4) {
1236 ifp->if_capenable ^= IFCAP_TSO4;
1239 if (mask & IFCAP_TSO6) {
1240 ifp->if_capenable ^= IFCAP_TSO6;
1243 if (mask & IFCAP_VLAN_HWTAGGING) {
1244 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1247 if (mask & IFCAP_VLAN_HWFILTER) {
1248 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1251 if (mask & IFCAP_VLAN_HWTSO) {
1252 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1255 if (mask & IFCAP_LRO) {
1256 ifp->if_capenable ^= IFCAP_LRO;
1259 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1261 VLAN_CAPABILITIES(ifp);
1266 error = ether_ioctl(ifp, command, data);
1274 /*********************************************************************
1277 * This routine is used in two ways. It is used by the stack as
1278 * init entry point in network interface structure. It is also used
1279 * by the driver as a hw/sw initialization routine to get to a
1282 * return 0 on success, positive on failure
1283 **********************************************************************/
1286 igb_init_locked(struct adapter *adapter)
1288 struct ifnet *ifp = adapter->ifp;
1289 device_t dev = adapter->dev;
1291 INIT_DEBUGOUT("igb_init: begin");
1293 IGB_CORE_LOCK_ASSERT(adapter);
1295 igb_disable_intr(adapter);
1296 callout_stop(&adapter->timer);
1298 /* Get the latest mac address, User can use a LAA */
1299 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1302 /* Put the address into the Receive Address Array */
1303 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1306 igb_update_link_status(adapter);
1308 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1310 /* Set hardware offload abilities */
1311 ifp->if_hwassist = 0;
1312 if (ifp->if_capenable & IFCAP_TXCSUM) {
1313 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1314 #if __FreeBSD_version >= 800000
1315 if (adapter->hw.mac.type == e1000_82576)
1316 ifp->if_hwassist |= CSUM_SCTP;
1320 if (ifp->if_capenable & IFCAP_TSO)
1321 ifp->if_hwassist |= CSUM_TSO;
1323 /* Configure for OS presence */
1324 igb_init_manageability(adapter);
1326 /* Prepare transmit descriptors and buffers */
1327 igb_setup_transmit_structures(adapter);
1328 igb_initialize_transmit_units(adapter);
1330 /* Setup Multicast table */
1331 igb_set_multi(adapter);
1334 ** Figure out the desired mbuf pool
1335 ** for doing jumbo/packetsplit
1337 if (adapter->max_frame_size <= 2048)
1338 adapter->rx_mbuf_sz = MCLBYTES;
1339 else if (adapter->max_frame_size <= 4096)
1340 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1342 adapter->rx_mbuf_sz = MJUM9BYTES;
1344 /* Prepare receive descriptors and buffers */
1345 if (igb_setup_receive_structures(adapter)) {
1346 device_printf(dev, "Could not setup receive structures\n");
1349 igb_initialize_receive_units(adapter);
1351 /* Enable VLAN support */
1352 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1353 igb_setup_vlan_hw_support(adapter);
1355 /* Don't lose promiscuous settings */
1356 igb_set_promisc(adapter);
1358 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1359 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1361 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1362 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1364 if (adapter->msix > 1) /* Set up queue routing */
1365 igb_configure_queues(adapter);
1367 /* this clears any pending interrupts */
1368 E1000_READ_REG(&adapter->hw, E1000_ICR);
1369 #ifdef DEVICE_POLLING
1371 * Only enable interrupts if we are not polling, make sure
1372 * they are off otherwise.
1374 if (ifp->if_capenable & IFCAP_POLLING)
1375 igb_disable_intr(adapter);
1377 #endif /* DEVICE_POLLING */
1379 igb_enable_intr(adapter);
1380 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1383 /* Set Energy Efficient Ethernet */
1384 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
1385 if (adapter->hw.mac.type == e1000_i354)
1386 e1000_set_eee_i354(&adapter->hw);
1388 e1000_set_eee_i350(&adapter->hw);
1395 struct adapter *adapter = arg;
1397 IGB_CORE_LOCK(adapter);
1398 igb_init_locked(adapter);
1399 IGB_CORE_UNLOCK(adapter);
1404 igb_handle_que(void *context, int pending)
1406 struct igb_queue *que = context;
1407 struct adapter *adapter = que->adapter;
1408 struct tx_ring *txr = que->txr;
1409 struct ifnet *ifp = adapter->ifp;
1411 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1414 more = igb_rxeof(que, adapter->rx_process_limit, NULL);
1418 #ifndef IGB_LEGACY_TX
1419 /* Process the stack queue only if not depleted */
1420 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1421 !drbr_empty(ifp, txr->br))
1422 igb_mq_start_locked(ifp, txr);
1424 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1425 igb_start_locked(txr, ifp);
1428 /* Do we need another? */
1430 taskqueue_enqueue(que->tq, &que->que_task);
1435 #ifdef DEVICE_POLLING
1436 if (ifp->if_capenable & IFCAP_POLLING)
1439 /* Reenable this interrupt */
1441 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1443 igb_enable_intr(adapter);
1446 /* Deal with link in a sleepable context */
1448 igb_handle_link(void *context, int pending)
1450 struct adapter *adapter = context;
1452 IGB_CORE_LOCK(adapter);
1453 igb_handle_link_locked(adapter);
1454 IGB_CORE_UNLOCK(adapter);
1458 igb_handle_link_locked(struct adapter *adapter)
1460 struct tx_ring *txr = adapter->tx_rings;
1461 struct ifnet *ifp = adapter->ifp;
1463 IGB_CORE_LOCK_ASSERT(adapter);
1464 adapter->hw.mac.get_link_status = 1;
1465 igb_update_link_status(adapter);
1466 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
1467 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1469 #ifndef IGB_LEGACY_TX
1470 /* Process the stack queue only if not depleted */
1471 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1472 !drbr_empty(ifp, txr->br))
1473 igb_mq_start_locked(ifp, txr);
1475 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1476 igb_start_locked(txr, ifp);
1483 /*********************************************************************
1485 * MSI/Legacy Deferred
1486 * Interrupt Service routine
1488 *********************************************************************/
1490 igb_irq_fast(void *arg)
1492 struct adapter *adapter = arg;
1493 struct igb_queue *que = adapter->queues;
1497 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1500 if (reg_icr == 0xffffffff)
1501 return FILTER_STRAY;
1503 /* Definitely not our interrupt. */
1505 return FILTER_STRAY;
1507 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1508 return FILTER_STRAY;
1511 * Mask interrupts until the taskqueue is finished running. This is
1512 * cheap, just assume that it is needed. This also works around the
1513 * MSI message reordering errata on certain systems.
1515 igb_disable_intr(adapter);
1516 taskqueue_enqueue(que->tq, &que->que_task);
1518 /* Link status change */
1519 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1520 taskqueue_enqueue(que->tq, &adapter->link_task);
1522 if (reg_icr & E1000_ICR_RXO)
1523 adapter->rx_overruns++;
1524 return FILTER_HANDLED;
1527 #ifdef DEVICE_POLLING
1528 #if __FreeBSD_version >= 800000
1529 #define POLL_RETURN_COUNT(a) (a)
1532 #define POLL_RETURN_COUNT(a)
1535 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1537 struct adapter *adapter = ifp->if_softc;
1538 struct igb_queue *que;
1539 struct tx_ring *txr;
1540 u32 reg_icr, rx_done = 0;
1541 u32 loop = IGB_MAX_LOOP;
1544 IGB_CORE_LOCK(adapter);
1545 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1546 IGB_CORE_UNLOCK(adapter);
1547 return POLL_RETURN_COUNT(rx_done);
1550 if (cmd == POLL_AND_CHECK_STATUS) {
1551 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1552 /* Link status change */
1553 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1554 igb_handle_link_locked(adapter);
1556 if (reg_icr & E1000_ICR_RXO)
1557 adapter->rx_overruns++;
1559 IGB_CORE_UNLOCK(adapter);
1561 for (int i = 0; i < adapter->num_queues; i++) {
1562 que = &adapter->queues[i];
1565 igb_rxeof(que, count, &rx_done);
1569 more = igb_txeof(txr);
1570 } while (loop-- && more);
1571 #ifndef IGB_LEGACY_TX
1572 if (!drbr_empty(ifp, txr->br))
1573 igb_mq_start_locked(ifp, txr);
1575 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1576 igb_start_locked(txr, ifp);
1581 return POLL_RETURN_COUNT(rx_done);
1583 #endif /* DEVICE_POLLING */
1585 /*********************************************************************
1587 * MSIX Que Interrupt Service routine
1589 **********************************************************************/
1591 igb_msix_que(void *arg)
1593 struct igb_queue *que = arg;
1594 struct adapter *adapter = que->adapter;
1595 struct ifnet *ifp = adapter->ifp;
1596 struct tx_ring *txr = que->txr;
1597 struct rx_ring *rxr = que->rxr;
1601 /* Ignore spurious interrupts */
1602 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1605 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1610 #ifndef IGB_LEGACY_TX
1611 /* Process the stack queue only if not depleted */
1612 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1613 !drbr_empty(ifp, txr->br))
1614 igb_mq_start_locked(ifp, txr);
1616 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1617 igb_start_locked(txr, ifp);
1621 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
1623 if (adapter->enable_aim == FALSE)
1626 ** Do Adaptive Interrupt Moderation:
1627 ** - Write out last calculated setting
1628 ** - Calculate based on average size over
1629 ** the last interval.
1631 if (que->eitr_setting)
1632 E1000_WRITE_REG(&adapter->hw,
1633 E1000_EITR(que->msix), que->eitr_setting);
1635 que->eitr_setting = 0;
1637 /* Idle, do nothing */
1638 if ((txr->bytes == 0) && (rxr->bytes == 0))
1641 /* Used half Default if sub-gig */
1642 if (adapter->link_speed != 1000)
1643 newitr = IGB_DEFAULT_ITR / 2;
1645 if ((txr->bytes) && (txr->packets))
1646 newitr = txr->bytes/txr->packets;
1647 if ((rxr->bytes) && (rxr->packets))
1648 newitr = max(newitr,
1649 (rxr->bytes / rxr->packets));
1650 newitr += 24; /* account for hardware frame, crc */
1651 /* set an upper boundary */
1652 newitr = min(newitr, 3000);
1653 /* Be nice to the mid range */
1654 if ((newitr > 300) && (newitr < 1200))
1655 newitr = (newitr / 3);
1657 newitr = (newitr / 2);
1659 newitr &= 0x7FFC; /* Mask invalid bits */
1660 if (adapter->hw.mac.type == e1000_82575)
1661 newitr |= newitr << 16;
1663 newitr |= E1000_EITR_CNT_IGNR;
1665 /* save for next interrupt */
1666 que->eitr_setting = newitr;
1675 /* Schedule a clean task if needed*/
1677 taskqueue_enqueue(que->tq, &que->que_task);
1679 /* Reenable this interrupt */
1680 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1685 /*********************************************************************
1687 * MSIX Link Interrupt Service routine
1689 **********************************************************************/
1692 igb_msix_link(void *arg)
1694 struct adapter *adapter = arg;
1697 ++adapter->link_irq;
1698 icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1699 if (!(icr & E1000_ICR_LSC))
1701 igb_handle_link(adapter, 0);
1705 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1706 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1711 /*********************************************************************
1713 * Media Ioctl callback
1715 * This routine is called whenever the user queries the status of
1716 * the interface using ifconfig.
1718 **********************************************************************/
1720 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1722 struct adapter *adapter = ifp->if_softc;
1724 INIT_DEBUGOUT("igb_media_status: begin");
1726 IGB_CORE_LOCK(adapter);
1727 igb_update_link_status(adapter);
1729 ifmr->ifm_status = IFM_AVALID;
1730 ifmr->ifm_active = IFM_ETHER;
1732 if (!adapter->link_active) {
1733 IGB_CORE_UNLOCK(adapter);
1737 ifmr->ifm_status |= IFM_ACTIVE;
1739 switch (adapter->link_speed) {
1741 ifmr->ifm_active |= IFM_10_T;
1745 ** Support for 100Mb SFP - these are Fiber
1746 ** but the media type appears as serdes
1748 if (adapter->hw.phy.media_type ==
1749 e1000_media_type_internal_serdes)
1750 ifmr->ifm_active |= IFM_100_FX;
1752 ifmr->ifm_active |= IFM_100_TX;
1755 ifmr->ifm_active |= IFM_1000_T;
1758 ifmr->ifm_active |= IFM_2500_SX;
1762 if (adapter->link_duplex == FULL_DUPLEX)
1763 ifmr->ifm_active |= IFM_FDX;
1765 ifmr->ifm_active |= IFM_HDX;
1767 IGB_CORE_UNLOCK(adapter);
1770 /*********************************************************************
1772 * Media Ioctl callback
1774 * This routine is called when the user changes speed/duplex using
1775 * media/mediopt option with ifconfig.
1777 **********************************************************************/
1779 igb_media_change(struct ifnet *ifp)
1781 struct adapter *adapter = ifp->if_softc;
1782 struct ifmedia *ifm = &adapter->media;
1784 INIT_DEBUGOUT("igb_media_change: begin");
1786 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1789 IGB_CORE_LOCK(adapter);
1790 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1792 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1793 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1798 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1799 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1802 adapter->hw.mac.autoneg = FALSE;
1803 adapter->hw.phy.autoneg_advertised = 0;
1804 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1805 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1807 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1810 adapter->hw.mac.autoneg = FALSE;
1811 adapter->hw.phy.autoneg_advertised = 0;
1812 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1813 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1815 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1818 device_printf(adapter->dev, "Unsupported media type\n");
1821 igb_init_locked(adapter);
1822 IGB_CORE_UNLOCK(adapter);
1828 /*********************************************************************
1830 * This routine maps the mbufs to Advanced TX descriptors.
1832 **********************************************************************/
1834 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1836 struct adapter *adapter = txr->adapter;
1837 u32 olinfo_status = 0, cmd_type_len;
1838 int i, j, error, nsegs;
1841 struct mbuf *m_head;
1842 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1844 struct igb_tx_buf *txbuf;
1845 union e1000_adv_tx_desc *txd = NULL;
1849 /* Basic descriptor defines */
1850 cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
1851 E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
1853 if (m_head->m_flags & M_VLANTAG)
1854 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1857 * Important to capture the first descriptor
1858 * used because it will contain the index of
1859 * the one we tell the hardware to report back
1861 first = txr->next_avail_desc;
1862 txbuf = &txr->tx_buffers[first];
1866 * Map the packet for DMA.
1869 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1870 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1872 if (__predict_false(error)) {
1877 /* Try it again? - one try */
1878 if (remap == TRUE) {
1880 m = m_defrag(*m_headp, M_NOWAIT);
1882 adapter->mbuf_defrag_failed++;
1892 txr->no_tx_dma_setup++;
1895 txr->no_tx_dma_setup++;
1902 /* Make certain there are enough descriptors */
1903 if (nsegs > txr->tx_avail - 2) {
1904 txr->no_desc_avail++;
1905 bus_dmamap_unload(txr->txtag, map);
1911 ** Set up the appropriate offload context
1912 ** this will consume the first descriptor
1914 error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1915 if (__predict_false(error)) {
1921 /* 82575 needs the queue index added */
1922 if (adapter->hw.mac.type == e1000_82575)
1923 olinfo_status |= txr->me << 4;
1925 i = txr->next_avail_desc;
1926 for (j = 0; j < nsegs; j++) {
1930 txbuf = &txr->tx_buffers[i];
1931 txd = &txr->tx_base[i];
1932 seglen = segs[j].ds_len;
1933 segaddr = htole64(segs[j].ds_addr);
1935 txd->read.buffer_addr = segaddr;
1936 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
1937 cmd_type_len | seglen);
1938 txd->read.olinfo_status = htole32(olinfo_status);
1940 if (++i == txr->num_desc)
1944 txd->read.cmd_type_len |=
1945 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1946 txr->tx_avail -= nsegs;
1947 txr->next_avail_desc = i;
1949 txbuf->m_head = m_head;
1951 ** Here we swap the map so the last descriptor,
1952 ** which gets the completion interrupt has the
1953 ** real map, and the first descriptor gets the
1954 ** unused map from this descriptor.
1956 txr->tx_buffers[first].map = txbuf->map;
1958 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1960 /* Set the EOP descriptor that will be marked done */
1961 txbuf = &txr->tx_buffers[first];
1964 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1965 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1967 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1968 * hardware that this frame is available to transmit.
1970 ++txr->total_packets;
1971 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1976 igb_set_promisc(struct adapter *adapter)
1978 struct ifnet *ifp = adapter->ifp;
1979 struct e1000_hw *hw = &adapter->hw;
1982 if (adapter->vf_ifp) {
1983 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
1987 reg = E1000_READ_REG(hw, E1000_RCTL);
1988 if (ifp->if_flags & IFF_PROMISC) {
1989 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1990 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1991 } else if (ifp->if_flags & IFF_ALLMULTI) {
1992 reg |= E1000_RCTL_MPE;
1993 reg &= ~E1000_RCTL_UPE;
1994 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1999 igb_disable_promisc(struct adapter *adapter)
2001 struct e1000_hw *hw = &adapter->hw;
2002 struct ifnet *ifp = adapter->ifp;
2006 if (adapter->vf_ifp) {
2007 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
2010 reg = E1000_READ_REG(hw, E1000_RCTL);
2011 reg &= (~E1000_RCTL_UPE);
2012 if (ifp->if_flags & IFF_ALLMULTI)
2013 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2015 struct ifmultiaddr *ifma;
2016 #if __FreeBSD_version < 800000
2019 if_maddr_rlock(ifp);
2021 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2022 if (ifma->ifma_addr->sa_family != AF_LINK)
2024 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2028 #if __FreeBSD_version < 800000
2029 IF_ADDR_UNLOCK(ifp);
2031 if_maddr_runlock(ifp);
2034 /* Don't disable if in MAX groups */
2035 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2036 reg &= (~E1000_RCTL_MPE);
2037 E1000_WRITE_REG(hw, E1000_RCTL, reg);
2041 /*********************************************************************
2044 * This routine is called whenever multicast address list is updated.
2046 **********************************************************************/
2049 igb_set_multi(struct adapter *adapter)
2051 struct ifnet *ifp = adapter->ifp;
2052 struct ifmultiaddr *ifma;
2058 IOCTL_DEBUGOUT("igb_set_multi: begin");
2061 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
2062 MAX_NUM_MULTICAST_ADDRESSES);
2064 #if __FreeBSD_version < 800000
2067 if_maddr_rlock(ifp);
2069 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2070 if (ifma->ifma_addr->sa_family != AF_LINK)
2073 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2076 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2077 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2080 #if __FreeBSD_version < 800000
2081 IF_ADDR_UNLOCK(ifp);
2083 if_maddr_runlock(ifp);
2086 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2087 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2088 reg_rctl |= E1000_RCTL_MPE;
2089 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2091 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2095 /*********************************************************************
2097 * This routine checks for link status,
2098 * updates statistics, and does the watchdog.
2100 **********************************************************************/
2103 igb_local_timer(void *arg)
2105 struct adapter *adapter = arg;
2106 device_t dev = adapter->dev;
2107 struct ifnet *ifp = adapter->ifp;
2108 struct tx_ring *txr = adapter->tx_rings;
2109 struct igb_queue *que = adapter->queues;
2110 int hung = 0, busy = 0;
2113 IGB_CORE_LOCK_ASSERT(adapter);
2115 igb_update_link_status(adapter);
2116 igb_update_stats_counters(adapter);
2119 ** Check the TX queues status
2120 ** - central locked handling of OACTIVE
2121 ** - watchdog only if all queues show hung
2123 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2124 if ((txr->queue_status & IGB_QUEUE_HUNG) &&
2125 (adapter->pause_frames == 0))
2127 if (txr->queue_status & IGB_QUEUE_DEPLETED)
2129 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
2130 taskqueue_enqueue(que->tq, &que->que_task);
2132 if (hung == adapter->num_queues)
2134 if (busy == adapter->num_queues)
2135 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2136 else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
2137 (busy < adapter->num_queues))
2138 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2140 adapter->pause_frames = 0;
2141 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
2142 #ifndef DEVICE_POLLING
2143 /* Schedule all queue interrupts - deadlock protection */
2144 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
2149 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2150 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2151 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2152 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2153 device_printf(dev,"TX(%d) desc avail = %d,"
2154 "Next TX to Clean = %d\n",
2155 txr->me, txr->tx_avail, txr->next_to_clean);
2156 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2157 adapter->watchdog_events++;
2158 igb_init_locked(adapter);
2162 igb_update_link_status(struct adapter *adapter)
2164 struct e1000_hw *hw = &adapter->hw;
2165 struct e1000_fc_info *fc = &hw->fc;
2166 struct ifnet *ifp = adapter->ifp;
2167 device_t dev = adapter->dev;
2168 struct tx_ring *txr = adapter->tx_rings;
2169 u32 link_check, thstat, ctrl;
2170 char *flowctl = NULL;
2172 link_check = thstat = ctrl = 0;
2174 /* Get the cached link value or read for real */
2175 switch (hw->phy.media_type) {
2176 case e1000_media_type_copper:
2177 if (hw->mac.get_link_status) {
2178 /* Do the work to read phy */
2179 e1000_check_for_link(hw);
2180 link_check = !hw->mac.get_link_status;
2184 case e1000_media_type_fiber:
2185 e1000_check_for_link(hw);
2186 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2189 case e1000_media_type_internal_serdes:
2190 e1000_check_for_link(hw);
2191 link_check = adapter->hw.mac.serdes_has_link;
2193 /* VF device is type_unknown */
2194 case e1000_media_type_unknown:
2195 e1000_check_for_link(hw);
2196 link_check = !hw->mac.get_link_status;
2202 /* Check for thermal downshift or shutdown */
2203 if (hw->mac.type == e1000_i350) {
2204 thstat = E1000_READ_REG(hw, E1000_THSTAT);
2205 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
2208 /* Get the flow control for display */
2209 switch (fc->current_mode) {
2210 case e1000_fc_rx_pause:
2213 case e1000_fc_tx_pause:
2225 /* Now we check if a transition has happened */
2226 if (link_check && (adapter->link_active == 0)) {
2227 e1000_get_speed_and_duplex(&adapter->hw,
2228 &adapter->link_speed, &adapter->link_duplex);
2230 device_printf(dev, "Link is up %d Mbps %s,"
2231 " Flow Control: %s\n",
2232 adapter->link_speed,
2233 ((adapter->link_duplex == FULL_DUPLEX) ?
2234 "Full Duplex" : "Half Duplex"), flowctl);
2235 adapter->link_active = 1;
2236 ifp->if_baudrate = adapter->link_speed * 1000000;
2237 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2238 (thstat & E1000_THSTAT_LINK_THROTTLE))
2239 device_printf(dev, "Link: thermal downshift\n");
2240 /* Delay Link Up for Phy update */
2241 if (((hw->mac.type == e1000_i210) ||
2242 (hw->mac.type == e1000_i211)) &&
2243 (hw->phy.id == I210_I_PHY_ID))
2244 msec_delay(I210_LINK_DELAY);
2245 /* Reset if the media type changed. */
2246 if (hw->dev_spec._82575.media_changed) {
2247 hw->dev_spec._82575.media_changed = false;
2248 adapter->flags |= IGB_MEDIA_RESET;
2251 /* This can sleep */
2252 if_link_state_change(ifp, LINK_STATE_UP);
2253 } else if (!link_check && (adapter->link_active == 1)) {
2254 ifp->if_baudrate = adapter->link_speed = 0;
2255 adapter->link_duplex = 0;
2257 device_printf(dev, "Link is Down\n");
2258 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2259 (thstat & E1000_THSTAT_PWR_DOWN))
2260 device_printf(dev, "Link: thermal shutdown\n");
2261 adapter->link_active = 0;
2262 /* This can sleep */
2263 if_link_state_change(ifp, LINK_STATE_DOWN);
2264 /* Reset queue state */
2265 for (int i = 0; i < adapter->num_queues; i++, txr++)
2266 txr->queue_status = IGB_QUEUE_IDLE;
2270 /*********************************************************************
2272 * This routine disables all traffic on the adapter by issuing a
2273 * global reset on the MAC and deallocates TX/RX buffers.
2275 **********************************************************************/
2280 struct adapter *adapter = arg;
2281 struct ifnet *ifp = adapter->ifp;
2282 struct tx_ring *txr = adapter->tx_rings;
2284 IGB_CORE_LOCK_ASSERT(adapter);
2286 INIT_DEBUGOUT("igb_stop: begin");
2288 igb_disable_intr(adapter);
2290 callout_stop(&adapter->timer);
2292 /* Tell the stack that the interface is no longer active */
2293 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2294 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2296 /* Disarm watchdog timer. */
2297 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2299 txr->queue_status = IGB_QUEUE_IDLE;
2303 e1000_reset_hw(&adapter->hw);
2304 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2306 e1000_led_off(&adapter->hw);
2307 e1000_cleanup_led(&adapter->hw);
2311 /*********************************************************************
2313 * Determine hardware revision.
2315 **********************************************************************/
2317 igb_identify_hardware(struct adapter *adapter)
2319 device_t dev = adapter->dev;
2321 /* Make sure our PCI config space has the necessary stuff set */
2322 pci_enable_busmaster(dev);
2323 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2325 /* Save off the information about this board */
2326 adapter->hw.vendor_id = pci_get_vendor(dev);
2327 adapter->hw.device_id = pci_get_device(dev);
2328 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2329 adapter->hw.subsystem_vendor_id =
2330 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2331 adapter->hw.subsystem_device_id =
2332 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2334 /* Set MAC type early for PCI setup */
2335 e1000_set_mac_type(&adapter->hw);
2337 /* Are we a VF device? */
2338 if ((adapter->hw.mac.type == e1000_vfadapt) ||
2339 (adapter->hw.mac.type == e1000_vfadapt_i350))
2340 adapter->vf_ifp = 1;
2342 adapter->vf_ifp = 0;
2346 igb_allocate_pci_resources(struct adapter *adapter)
2348 device_t dev = adapter->dev;
2352 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2354 if (adapter->pci_mem == NULL) {
2355 device_printf(dev, "Unable to allocate bus resource: memory\n");
2358 adapter->osdep.mem_bus_space_tag =
2359 rman_get_bustag(adapter->pci_mem);
2360 adapter->osdep.mem_bus_space_handle =
2361 rman_get_bushandle(adapter->pci_mem);
2362 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2364 adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2366 /* This will setup either MSI/X or MSI */
2367 adapter->msix = igb_setup_msix(adapter);
2368 adapter->hw.back = &adapter->osdep;
2373 /*********************************************************************
2375 * Setup the Legacy or MSI Interrupt handler
2377 **********************************************************************/
2379 igb_allocate_legacy(struct adapter *adapter)
2381 device_t dev = adapter->dev;
2382 struct igb_queue *que = adapter->queues;
2383 struct tx_ring *txr = adapter->tx_rings;
2386 /* Turn off all interrupts */
2387 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2390 if (adapter->msix == 1)
2393 /* We allocate a single interrupt resource */
2394 adapter->res = bus_alloc_resource_any(dev,
2395 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2396 if (adapter->res == NULL) {
2397 device_printf(dev, "Unable to allocate bus resource: "
2402 #ifndef IGB_LEGACY_TX
2403 TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
2407 * Try allocating a fast interrupt and the associated deferred
2408 * processing contexts.
2410 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2411 /* Make tasklet for deferred link handling */
2412 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2413 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2414 taskqueue_thread_enqueue, &que->tq);
2415 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
2416 device_get_nameunit(adapter->dev));
2417 if ((error = bus_setup_intr(dev, adapter->res,
2418 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2419 adapter, &adapter->tag)) != 0) {
2420 device_printf(dev, "Failed to register fast interrupt "
2421 "handler: %d\n", error);
2422 taskqueue_free(que->tq);
2431 /*********************************************************************
2433 * Setup the MSIX Queue Interrupt handlers:
2435 **********************************************************************/
2437 igb_allocate_msix(struct adapter *adapter)
2439 device_t dev = adapter->dev;
2440 struct igb_queue *que = adapter->queues;
2441 int error, rid, vector = 0;
2443 /* Be sure to start with all interrupts disabled */
2444 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
2445 E1000_WRITE_FLUSH(&adapter->hw);
2447 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2449 que->res = bus_alloc_resource_any(dev,
2450 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2451 if (que->res == NULL) {
2453 "Unable to allocate bus resource: "
2454 "MSIX Queue Interrupt\n");
2457 error = bus_setup_intr(dev, que->res,
2458 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2459 igb_msix_que, que, &que->tag);
2462 device_printf(dev, "Failed to register Queue handler");
2465 #if __FreeBSD_version >= 800504
2466 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2469 if (adapter->hw.mac.type == e1000_82575)
2470 que->eims = E1000_EICR_TX_QUEUE0 << i;
2472 que->eims = 1 << vector;
2474 ** Bind the msix vector, and thus the
2475 ** rings to the corresponding cpu.
2477 if (adapter->num_queues > 1) {
2478 if (igb_last_bind_cpu < 0)
2479 igb_last_bind_cpu = CPU_FIRST();
2480 bus_bind_intr(dev, que->res, igb_last_bind_cpu);
2482 "Bound queue %d to cpu %d\n",
2483 i,igb_last_bind_cpu);
2484 igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
2486 #ifndef IGB_LEGACY_TX
2487 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
2490 /* Make tasklet for deferred handling */
2491 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2492 que->tq = taskqueue_create("igb_que", M_NOWAIT,
2493 taskqueue_thread_enqueue, &que->tq);
2494 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2495 device_get_nameunit(adapter->dev));
2500 adapter->res = bus_alloc_resource_any(dev,
2501 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2502 if (adapter->res == NULL) {
2504 "Unable to allocate bus resource: "
2505 "MSIX Link Interrupt\n");
2508 if ((error = bus_setup_intr(dev, adapter->res,
2509 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2510 igb_msix_link, adapter, &adapter->tag)) != 0) {
2511 device_printf(dev, "Failed to register Link handler");
2514 #if __FreeBSD_version >= 800504
2515 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2517 adapter->linkvec = vector;
2524 igb_configure_queues(struct adapter *adapter)
2526 struct e1000_hw *hw = &adapter->hw;
2527 struct igb_queue *que;
2528 u32 tmp, ivar = 0, newitr = 0;
2530 /* First turn on RSS capability */
2531 if (adapter->hw.mac.type != e1000_82575)
2532 E1000_WRITE_REG(hw, E1000_GPIE,
2533 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2534 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2537 switch (adapter->hw.mac.type) {
2544 case e1000_vfadapt_i350:
2546 for (int i = 0; i < adapter->num_queues; i++) {
2548 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2549 que = &adapter->queues[i];
2552 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2555 ivar |= que->msix | E1000_IVAR_VALID;
2557 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2560 for (int i = 0; i < adapter->num_queues; i++) {
2562 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2563 que = &adapter->queues[i];
2566 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2569 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2571 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2572 adapter->que_mask |= que->eims;
2575 /* And for the link interrupt */
2576 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2577 adapter->link_mask = 1 << adapter->linkvec;
2578 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2582 for (int i = 0; i < adapter->num_queues; i++) {
2583 u32 index = i & 0x7; /* Each IVAR has two entries */
2584 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2585 que = &adapter->queues[i];
2588 ivar |= que->msix | E1000_IVAR_VALID;
2591 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2593 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2594 adapter->que_mask |= que->eims;
2597 for (int i = 0; i < adapter->num_queues; i++) {
2598 u32 index = i & 0x7; /* Each IVAR has two entries */
2599 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2600 que = &adapter->queues[i];
2603 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2606 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2608 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2609 adapter->que_mask |= que->eims;
2612 /* And for the link interrupt */
2613 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2614 adapter->link_mask = 1 << adapter->linkvec;
2615 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2619 /* enable MSI-X support*/
2620 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2621 tmp |= E1000_CTRL_EXT_PBA_CLR;
2622 /* Auto-Mask interrupts upon ICR read. */
2623 tmp |= E1000_CTRL_EXT_EIAME;
2624 tmp |= E1000_CTRL_EXT_IRCA;
2625 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2628 for (int i = 0; i < adapter->num_queues; i++) {
2629 que = &adapter->queues[i];
2630 tmp = E1000_EICR_RX_QUEUE0 << i;
2631 tmp |= E1000_EICR_TX_QUEUE0 << i;
2633 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2635 adapter->que_mask |= que->eims;
2639 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2641 adapter->link_mask |= E1000_EIMS_OTHER;
2646 /* Set the starting interrupt rate */
2647 if (igb_max_interrupt_rate > 0)
2648 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
2650 if (hw->mac.type == e1000_82575)
2651 newitr |= newitr << 16;
2653 newitr |= E1000_EITR_CNT_IGNR;
2655 for (int i = 0; i < adapter->num_queues; i++) {
2656 que = &adapter->queues[i];
2657 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2665 igb_free_pci_resources(struct adapter *adapter)
2667 struct igb_queue *que = adapter->queues;
2668 device_t dev = adapter->dev;
2672 ** There is a slight possibility of a failure mode
2673 ** in attach that will result in entering this function
2674 ** before interrupt resources have been initialized, and
2675 ** in that case we do not want to execute the loops below
2676 ** We can detect this reliably by the state of the adapter
2679 if (adapter->res == NULL)
2683 * First release all the interrupt resources:
2685 for (int i = 0; i < adapter->num_queues; i++, que++) {
2686 rid = que->msix + 1;
2687 if (que->tag != NULL) {
2688 bus_teardown_intr(dev, que->res, que->tag);
2691 if (que->res != NULL)
2692 bus_release_resource(dev,
2693 SYS_RES_IRQ, rid, que->res);
2696 /* Clean the Legacy or Link interrupt last */
2697 if (adapter->linkvec) /* we are doing MSIX */
2698 rid = adapter->linkvec + 1;
2700 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2702 que = adapter->queues;
2703 if (adapter->tag != NULL) {
2704 taskqueue_drain(que->tq, &adapter->link_task);
2705 bus_teardown_intr(dev, adapter->res, adapter->tag);
2706 adapter->tag = NULL;
2708 if (adapter->res != NULL)
2709 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2711 for (int i = 0; i < adapter->num_queues; i++, que++) {
2712 if (que->tq != NULL) {
2713 #ifndef IGB_LEGACY_TX
2714 taskqueue_drain(que->tq, &que->txr->txq_task);
2716 taskqueue_drain(que->tq, &que->que_task);
2717 taskqueue_free(que->tq);
2722 pci_release_msi(dev);
2724 if (adapter->msix_mem != NULL)
2725 bus_release_resource(dev, SYS_RES_MEMORY,
2726 adapter->memrid, adapter->msix_mem);
2728 if (adapter->pci_mem != NULL)
2729 bus_release_resource(dev, SYS_RES_MEMORY,
2730 PCIR_BAR(0), adapter->pci_mem);
2735 * Setup Either MSI/X or MSI
2738 igb_setup_msix(struct adapter *adapter)
2740 device_t dev = adapter->dev;
2741 int bar, want, queues, msgs, maxqueues;
2743 /* tuneable override */
2744 if (igb_enable_msix == 0)
2747 /* First try MSI/X */
2748 msgs = pci_msix_count(dev);
2752 ** Some new devices, as with ixgbe, now may
2753 ** use a different BAR, so we need to keep
2754 ** track of which is used.
2756 adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
2757 bar = pci_read_config(dev, adapter->memrid, 4);
2758 if (bar == 0) /* use next bar */
2759 adapter->memrid += 4;
2760 adapter->msix_mem = bus_alloc_resource_any(dev,
2761 SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
2762 if (adapter->msix_mem == NULL) {
2763 /* May not be enabled */
2764 device_printf(adapter->dev,
2765 "Unable to map MSIX table \n");
2769 /* Figure out a reasonable auto config value */
2770 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2772 /* Manual override */
2773 if (igb_num_queues != 0)
2774 queues = igb_num_queues;
2776 /* Sanity check based on HW */
2777 switch (adapter->hw.mac.type) {
2793 default: /* VF interfaces */
2797 if (queues > maxqueues)
2800 /* Manual override */
2801 if (igb_num_queues != 0)
2802 queues = igb_num_queues;
2805 ** One vector (RX/TX pair) per queue
2806 ** plus an additional for Link interrupt
2812 device_printf(adapter->dev,
2813 "MSIX Configuration Problem, "
2814 "%d vectors configured, but %d queues wanted!\n",
2818 if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2819 device_printf(adapter->dev,
2820 "Using MSIX interrupts with %d vectors\n", msgs);
2821 adapter->num_queues = queues;
2825 ** If MSIX alloc failed or provided us with
2826 ** less than needed, free and fall through to MSI
2828 pci_release_msi(dev);
2831 if (adapter->msix_mem != NULL) {
2832 bus_release_resource(dev, SYS_RES_MEMORY,
2833 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2834 adapter->msix_mem = NULL;
2837 if (pci_alloc_msi(dev, &msgs) == 0) {
2838 device_printf(adapter->dev," Using an MSI interrupt\n");
2841 device_printf(adapter->dev," Using a Legacy interrupt\n");
2845 /*********************************************************************
2847 * Initialize the DMA Coalescing feature
2849 **********************************************************************/
2851 igb_init_dmac(struct adapter *adapter, u32 pba)
2853 device_t dev = adapter->dev;
2854 struct e1000_hw *hw = &adapter->hw;
2855 u32 dmac, reg = ~E1000_DMACR_DMAC_EN;
2858 if (hw->mac.type == e1000_i211)
2861 if (hw->mac.type > e1000_82580) {
2863 if (adapter->dmac == 0) { /* Disabling it */
2864 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2867 device_printf(dev, "DMA Coalescing enabled\n");
2869 /* Set starting threshold */
2870 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
2872 hwm = 64 * pba - adapter->max_frame_size / 16;
2873 if (hwm < 64 * (pba - 6))
2874 hwm = 64 * (pba - 6);
2875 reg = E1000_READ_REG(hw, E1000_FCRTC);
2876 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
2877 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
2878 & E1000_FCRTC_RTH_COAL_MASK);
2879 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
2882 dmac = pba - adapter->max_frame_size / 512;
2883 if (dmac < pba - 10)
2885 reg = E1000_READ_REG(hw, E1000_DMACR);
2886 reg &= ~E1000_DMACR_DMACTHR_MASK;
2887 reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
2888 & E1000_DMACR_DMACTHR_MASK);
2890 /* transition to L0x or L1 if available..*/
2891 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
2893 /* Check if status is 2.5Gb backplane connection
2894 * before configuration of watchdog timer, which is
2895 * in msec values in 12.8usec intervals
2896 * watchdog timer= msec values in 32usec intervals
2897 * for non 2.5Gb connection
2899 if (hw->mac.type == e1000_i354) {
2900 int status = E1000_READ_REG(hw, E1000_STATUS);
2901 if ((status & E1000_STATUS_2P5_SKU) &&
2902 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2903 reg |= ((adapter->dmac * 5) >> 6);
2905 reg |= (adapter->dmac >> 5);
2907 reg |= (adapter->dmac >> 5);
2910 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2912 #ifdef I210_OBFF_SUPPORT
2914 * Set the OBFF Rx threshold to DMA Coalescing Rx
2915 * threshold - 2KB and enable the feature in the
2916 * hardware for I210.
2918 if (hw->mac.type == e1000_i210) {
2919 int obff = dmac - 2;
2920 reg = E1000_READ_REG(hw, E1000_DOBFFCTL);
2921 reg &= ~E1000_DOBFFCTL_OBFFTHR_MASK;
2922 reg |= (obff & E1000_DOBFFCTL_OBFFTHR_MASK)
2923 | E1000_DOBFFCTL_EXIT_ACT_MASK;
2924 E1000_WRITE_REG(hw, E1000_DOBFFCTL, reg);
2927 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
2929 /* Set the interval before transition */
2930 reg = E1000_READ_REG(hw, E1000_DMCTLX);
2931 if (hw->mac.type == e1000_i350)
2932 reg |= IGB_DMCTLX_DCFLUSH_DIS;
2934 ** in 2.5Gb connection, TTLX unit is 0.4 usec
2935 ** which is 0x4*2 = 0xA. But delay is still 4 usec
2937 if (hw->mac.type == e1000_i354) {
2938 int status = E1000_READ_REG(hw, E1000_STATUS);
2939 if ((status & E1000_STATUS_2P5_SKU) &&
2940 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2948 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
2950 /* free space in tx packet buffer to wake from DMA coal */
2951 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
2952 (2 * adapter->max_frame_size)) >> 6);
2954 /* make low power state decision controlled by DMA coal */
2955 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2956 reg &= ~E1000_PCIEMISC_LX_DECISION;
2957 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
2959 } else if (hw->mac.type == e1000_82580) {
2960 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2961 E1000_WRITE_REG(hw, E1000_PCIEMISC,
2962 reg & ~E1000_PCIEMISC_LX_DECISION);
2963 E1000_WRITE_REG(hw, E1000_DMACR, 0);
2968 /*********************************************************************
2970 * Set up an fresh starting state
2972 **********************************************************************/
2974 igb_reset(struct adapter *adapter)
2976 device_t dev = adapter->dev;
2977 struct e1000_hw *hw = &adapter->hw;
2978 struct e1000_fc_info *fc = &hw->fc;
2979 struct ifnet *ifp = adapter->ifp;
2983 INIT_DEBUGOUT("igb_reset: begin");
2985 /* Let the firmware know the OS is in control */
2986 igb_get_hw_control(adapter);
2989 * Packet Buffer Allocation (PBA)
2990 * Writing PBA sets the receive portion of the buffer
2991 * the remainder is used for the transmit buffer.
2993 switch (hw->mac.type) {
2995 pba = E1000_PBA_32K;
2999 pba = E1000_READ_REG(hw, E1000_RXPBS);
3000 pba &= E1000_RXPBS_SIZE_MASK_82576;
3005 case e1000_vfadapt_i350:
3006 pba = E1000_READ_REG(hw, E1000_RXPBS);
3007 pba = e1000_rxpbs_adjust_82580(pba);
3011 pba = E1000_PBA_34K;
3016 /* Special needs in case of Jumbo frames */
3017 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
3018 u32 tx_space, min_tx, min_rx;
3019 pba = E1000_READ_REG(hw, E1000_PBA);
3020 tx_space = pba >> 16;
3022 min_tx = (adapter->max_frame_size +
3023 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
3024 min_tx = roundup2(min_tx, 1024);
3026 min_rx = adapter->max_frame_size;
3027 min_rx = roundup2(min_rx, 1024);
3029 if (tx_space < min_tx &&
3030 ((min_tx - tx_space) < pba)) {
3031 pba = pba - (min_tx - tx_space);
3033 * if short on rx space, rx wins
3034 * and must trump tx adjustment
3039 E1000_WRITE_REG(hw, E1000_PBA, pba);
3042 INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
3045 * These parameters control the automatic generation (Tx) and
3046 * response (Rx) to Ethernet PAUSE frames.
3047 * - High water mark should allow for at least two frames to be
3048 * received after sending an XOFF.
3049 * - Low water mark works best when it is very near the high water mark.
3050 * This allows the receiver to restart by sending XON when it has
3053 hwm = min(((pba << 10) * 9 / 10),
3054 ((pba << 10) - 2 * adapter->max_frame_size));
3056 if (hw->mac.type < e1000_82576) {
3057 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
3058 fc->low_water = fc->high_water - 8;
3060 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
3061 fc->low_water = fc->high_water - 16;
3064 fc->pause_time = IGB_FC_PAUSE_TIME;
3065 fc->send_xon = TRUE;
3067 fc->requested_mode = adapter->fc;
3069 fc->requested_mode = e1000_fc_default;
3071 /* Issue a global reset */
3073 E1000_WRITE_REG(hw, E1000_WUC, 0);
3075 /* Reset for AutoMediaDetect */
3076 if (adapter->flags & IGB_MEDIA_RESET) {
3077 e1000_setup_init_funcs(hw, TRUE);
3078 e1000_get_bus_info(hw);
3079 adapter->flags &= ~IGB_MEDIA_RESET;
3082 if (e1000_init_hw(hw) < 0)
3083 device_printf(dev, "Hardware Initialization Failed\n");
3085 /* Setup DMA Coalescing */
3086 igb_init_dmac(adapter, pba);
3088 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
3089 e1000_get_phy_info(hw);
3090 e1000_check_for_link(hw);
3094 /*********************************************************************
3096 * Setup networking device structure and register an interface.
3098 **********************************************************************/
3100 igb_setup_interface(device_t dev, struct adapter *adapter)
3104 INIT_DEBUGOUT("igb_setup_interface: begin");
3106 ifp = adapter->ifp = if_alloc(IFT_ETHER);
3108 device_printf(dev, "can not allocate ifnet structure\n");
3111 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3112 ifp->if_init = igb_init;
3113 ifp->if_softc = adapter;
3114 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3115 ifp->if_ioctl = igb_ioctl;
3116 #ifndef IGB_LEGACY_TX
3117 ifp->if_transmit = igb_mq_start;
3118 ifp->if_qflush = igb_qflush;
3120 ifp->if_start = igb_start;
3121 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3122 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3123 IFQ_SET_READY(&ifp->if_snd);
3126 ether_ifattach(ifp, adapter->hw.mac.addr);
3128 ifp->if_capabilities = ifp->if_capenable = 0;
3130 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3131 ifp->if_capabilities |= IFCAP_TSO;
3132 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
3133 ifp->if_capenable = ifp->if_capabilities;
3135 /* Don't enable LRO by default */
3136 ifp->if_capabilities |= IFCAP_LRO;
3138 #ifdef DEVICE_POLLING
3139 ifp->if_capabilities |= IFCAP_POLLING;
3143 * Tell the upper layer(s) we
3144 * support full VLAN capability.
3146 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3147 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
3150 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
3155 ** Don't turn this on by default, if vlans are
3156 ** created on another pseudo device (eg. lagg)
3157 ** then vlan events are not passed thru, breaking
3158 ** operation, but with HW FILTER off it works. If
3159 ** using vlans directly on the igb driver you can
3160 ** enable this and get full hardware tag filtering.
3162 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3165 * Specify the media types supported by this adapter and register
3166 * callbacks to update media and link information
3168 ifmedia_init(&adapter->media, IFM_IMASK,
3169 igb_media_change, igb_media_status);
3170 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3171 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3172 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
3174 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
3176 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3177 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3179 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3181 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3183 if (adapter->hw.phy.type != e1000_phy_ife) {
3184 ifmedia_add(&adapter->media,
3185 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3186 ifmedia_add(&adapter->media,
3187 IFM_ETHER | IFM_1000_T, 0, NULL);
3190 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3191 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3197 * Manage DMA'able memory.
3200 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3204 *(bus_addr_t *) arg = segs[0].ds_addr;
3208 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
3209 struct igb_dma_alloc *dma, int mapflags)
3213 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3214 IGB_DBA_ALIGN, 0, /* alignment, bounds */
3215 BUS_SPACE_MAXADDR, /* lowaddr */
3216 BUS_SPACE_MAXADDR, /* highaddr */
3217 NULL, NULL, /* filter, filterarg */
3220 size, /* maxsegsize */
3222 NULL, /* lockfunc */
3226 device_printf(adapter->dev,
3227 "%s: bus_dma_tag_create failed: %d\n",
3232 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3233 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3235 device_printf(adapter->dev,
3236 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3237 __func__, (uintmax_t)size, error);
3242 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3243 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3244 if (error || dma->dma_paddr == 0) {
3245 device_printf(adapter->dev,
3246 "%s: bus_dmamap_load failed: %d\n",
3254 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3256 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3257 bus_dma_tag_destroy(dma->dma_tag);
3259 dma->dma_map = NULL;
3260 dma->dma_tag = NULL;
3266 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
3268 if (dma->dma_tag == NULL)
3270 if (dma->dma_map != NULL) {
3271 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3272 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3273 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3274 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3275 dma->dma_map = NULL;
3277 bus_dma_tag_destroy(dma->dma_tag);
3278 dma->dma_tag = NULL;
3282 /*********************************************************************
3284 * Allocate memory for the transmit and receive rings, and then
3285 * the descriptors associated with each, called only once at attach.
3287 **********************************************************************/
3289 igb_allocate_queues(struct adapter *adapter)
3291 device_t dev = adapter->dev;
3292 struct igb_queue *que = NULL;
3293 struct tx_ring *txr = NULL;
3294 struct rx_ring *rxr = NULL;
3295 int rsize, tsize, error = E1000_SUCCESS;
3296 int txconf = 0, rxconf = 0;
3298 /* First allocate the top level queue structs */
3299 if (!(adapter->queues =
3300 (struct igb_queue *) malloc(sizeof(struct igb_queue) *
3301 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3302 device_printf(dev, "Unable to allocate queue memory\n");
3307 /* Next allocate the TX ring struct memory */
3308 if (!(adapter->tx_rings =
3309 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3310 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3311 device_printf(dev, "Unable to allocate TX ring memory\n");
3316 /* Now allocate the RX */
3317 if (!(adapter->rx_rings =
3318 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3319 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3320 device_printf(dev, "Unable to allocate RX ring memory\n");
3325 tsize = roundup2(adapter->num_tx_desc *
3326 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
3328 * Now set up the TX queues, txconf is needed to handle the
3329 * possibility that things fail midcourse and we need to
3330 * undo memory gracefully
3332 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3333 /* Set up some basics */
3334 txr = &adapter->tx_rings[i];
3335 txr->adapter = adapter;
3337 txr->num_desc = adapter->num_tx_desc;
3339 /* Initialize the TX lock */
3340 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3341 device_get_nameunit(dev), txr->me);
3342 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3344 if (igb_dma_malloc(adapter, tsize,
3345 &txr->txdma, BUS_DMA_NOWAIT)) {
3347 "Unable to allocate TX Descriptor memory\n");
3351 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
3352 bzero((void *)txr->tx_base, tsize);
3354 /* Now allocate transmit buffers for the ring */
3355 if (igb_allocate_transmit_buffers(txr)) {
3357 "Critical Failure setting up transmit buffers\n");
3361 #ifndef IGB_LEGACY_TX
3362 /* Allocate a buf ring */
3363 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
3364 M_WAITOK, &txr->tx_mtx);
3369 * Next the RX queues...
3371 rsize = roundup2(adapter->num_rx_desc *
3372 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3373 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3374 rxr = &adapter->rx_rings[i];
3375 rxr->adapter = adapter;
3378 /* Initialize the RX lock */
3379 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3380 device_get_nameunit(dev), txr->me);
3381 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3383 if (igb_dma_malloc(adapter, rsize,
3384 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3386 "Unable to allocate RxDescriptor memory\n");
3390 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3391 bzero((void *)rxr->rx_base, rsize);
3393 /* Allocate receive buffers for the ring*/
3394 if (igb_allocate_receive_buffers(rxr)) {
3396 "Critical Failure setting up receive buffers\n");
3403 ** Finally set up the queue holding structs
3405 for (int i = 0; i < adapter->num_queues; i++) {
3406 que = &adapter->queues[i];
3407 que->adapter = adapter;
3408 que->txr = &adapter->tx_rings[i];
3409 que->rxr = &adapter->rx_rings[i];
3415 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3416 igb_dma_free(adapter, &rxr->rxdma);
3418 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3419 igb_dma_free(adapter, &txr->txdma);
3420 free(adapter->rx_rings, M_DEVBUF);
3422 #ifndef IGB_LEGACY_TX
3423 buf_ring_free(txr->br, M_DEVBUF);
3425 free(adapter->tx_rings, M_DEVBUF);
3427 free(adapter->queues, M_DEVBUF);
3432 /*********************************************************************
3434 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3435 * the information needed to transmit a packet on the wire. This is
3436 * called only once at attach, setup is done every reset.
3438 **********************************************************************/
3440 igb_allocate_transmit_buffers(struct tx_ring *txr)
3442 struct adapter *adapter = txr->adapter;
3443 device_t dev = adapter->dev;
3444 struct igb_tx_buf *txbuf;
3448 * Setup DMA descriptor areas.
3450 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3451 1, 0, /* alignment, bounds */
3452 BUS_SPACE_MAXADDR, /* lowaddr */
3453 BUS_SPACE_MAXADDR, /* highaddr */
3454 NULL, NULL, /* filter, filterarg */
3455 IGB_TSO_SIZE, /* maxsize */
3456 IGB_MAX_SCATTER, /* nsegments */
3457 PAGE_SIZE, /* maxsegsize */
3459 NULL, /* lockfunc */
3460 NULL, /* lockfuncarg */
3462 device_printf(dev,"Unable to allocate TX DMA tag\n");
3466 if (!(txr->tx_buffers =
3467 (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
3468 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3469 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3474 /* Create the descriptor buffer dma maps */
3475 txbuf = txr->tx_buffers;
3476 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3477 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3479 device_printf(dev, "Unable to create TX DMA map\n");
3486 /* We free all, it handles case where we are in the middle */
3487 igb_free_transmit_structures(adapter);
3491 /*********************************************************************
3493 * Initialize a transmit ring.
3495 **********************************************************************/
3497 igb_setup_transmit_ring(struct tx_ring *txr)
3499 struct adapter *adapter = txr->adapter;
3500 struct igb_tx_buf *txbuf;
3503 struct netmap_adapter *na = NA(adapter->ifp);
3504 struct netmap_slot *slot;
3505 #endif /* DEV_NETMAP */
3507 /* Clear the old descriptor contents */
3510 slot = netmap_reset(na, NR_TX, txr->me, 0);
3511 #endif /* DEV_NETMAP */
3512 bzero((void *)txr->tx_base,
3513 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3515 txr->next_avail_desc = 0;
3516 txr->next_to_clean = 0;
3518 /* Free any existing tx buffers. */
3519 txbuf = txr->tx_buffers;
3520 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3521 if (txbuf->m_head != NULL) {
3522 bus_dmamap_sync(txr->txtag, txbuf->map,
3523 BUS_DMASYNC_POSTWRITE);
3524 bus_dmamap_unload(txr->txtag, txbuf->map);
3525 m_freem(txbuf->m_head);
3526 txbuf->m_head = NULL;
3530 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3531 /* no need to set the address */
3532 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3534 #endif /* DEV_NETMAP */
3535 /* clear the watch index */
3539 /* Set number of descriptors available */
3540 txr->tx_avail = adapter->num_tx_desc;
3542 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3543 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3547 /*********************************************************************
3549 * Initialize all transmit rings.
3551 **********************************************************************/
3553 igb_setup_transmit_structures(struct adapter *adapter)
3555 struct tx_ring *txr = adapter->tx_rings;
3557 for (int i = 0; i < adapter->num_queues; i++, txr++)
3558 igb_setup_transmit_ring(txr);
3563 /*********************************************************************
3565 * Enable transmit unit.
3567 **********************************************************************/
3569 igb_initialize_transmit_units(struct adapter *adapter)
3571 struct tx_ring *txr = adapter->tx_rings;
3572 struct e1000_hw *hw = &adapter->hw;
3575 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3578 /* Setup the Tx Descriptor Rings */
3579 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3580 u64 bus_addr = txr->txdma.dma_paddr;
3582 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3583 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3584 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3585 (uint32_t)(bus_addr >> 32));
3586 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3587 (uint32_t)bus_addr);
3589 /* Setup the HW Tx Head and Tail descriptor pointers */
3590 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3591 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3593 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3594 E1000_READ_REG(hw, E1000_TDBAL(i)),
3595 E1000_READ_REG(hw, E1000_TDLEN(i)));
3597 txr->queue_status = IGB_QUEUE_IDLE;
3599 txdctl |= IGB_TX_PTHRESH;
3600 txdctl |= IGB_TX_HTHRESH << 8;
3601 txdctl |= IGB_TX_WTHRESH << 16;
3602 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3603 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3606 if (adapter->vf_ifp)
3609 e1000_config_collision_dist(hw);
3611 /* Program the Transmit Control Register */
3612 tctl = E1000_READ_REG(hw, E1000_TCTL);
3613 tctl &= ~E1000_TCTL_CT;
3614 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3615 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3617 /* This write will effectively turn on the transmit unit. */
3618 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3621 /*********************************************************************
3623 * Free all transmit rings.
3625 **********************************************************************/
3627 igb_free_transmit_structures(struct adapter *adapter)
3629 struct tx_ring *txr = adapter->tx_rings;
3631 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3633 igb_free_transmit_buffers(txr);
3634 igb_dma_free(adapter, &txr->txdma);
3636 IGB_TX_LOCK_DESTROY(txr);
3638 free(adapter->tx_rings, M_DEVBUF);
3641 /*********************************************************************
3643 * Free transmit ring related data structures.
3645 **********************************************************************/
3647 igb_free_transmit_buffers(struct tx_ring *txr)
3649 struct adapter *adapter = txr->adapter;
3650 struct igb_tx_buf *tx_buffer;
3653 INIT_DEBUGOUT("free_transmit_ring: begin");
3655 if (txr->tx_buffers == NULL)
3658 tx_buffer = txr->tx_buffers;
3659 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3660 if (tx_buffer->m_head != NULL) {
3661 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3662 BUS_DMASYNC_POSTWRITE);
3663 bus_dmamap_unload(txr->txtag,
3665 m_freem(tx_buffer->m_head);
3666 tx_buffer->m_head = NULL;
3667 if (tx_buffer->map != NULL) {
3668 bus_dmamap_destroy(txr->txtag,
3670 tx_buffer->map = NULL;
3672 } else if (tx_buffer->map != NULL) {
3673 bus_dmamap_unload(txr->txtag,
3675 bus_dmamap_destroy(txr->txtag,
3677 tx_buffer->map = NULL;
3680 #ifndef IGB_LEGACY_TX
3681 if (txr->br != NULL)
3682 buf_ring_free(txr->br, M_DEVBUF);
3684 if (txr->tx_buffers != NULL) {
3685 free(txr->tx_buffers, M_DEVBUF);
3686 txr->tx_buffers = NULL;
3688 if (txr->txtag != NULL) {
3689 bus_dma_tag_destroy(txr->txtag);
3695 /**********************************************************************
3697 * Setup work for hardware segmentation offload (TSO) on
3698 * adapters using advanced tx descriptors
3700 **********************************************************************/
3702 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3703 u32 *cmd_type_len, u32 *olinfo_status)
3705 struct adapter *adapter = txr->adapter;
3706 struct e1000_adv_tx_context_desc *TXD;
3707 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3708 u32 mss_l4len_idx = 0, paylen;
3709 u16 vtag = 0, eh_type;
3710 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3711 struct ether_vlan_header *eh;
3713 struct ip6_hdr *ip6;
3722 * Determine where frame payload starts.
3723 * Jump over vlan headers if already present
3725 eh = mtod(mp, struct ether_vlan_header *);
3726 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3727 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3728 eh_type = eh->evl_proto;
3730 ehdrlen = ETHER_HDR_LEN;
3731 eh_type = eh->evl_encap_proto;
3734 switch (ntohs(eh_type)) {
3736 case ETHERTYPE_IPV6:
3737 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3738 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3739 if (ip6->ip6_nxt != IPPROTO_TCP)
3741 ip_hlen = sizeof(struct ip6_hdr);
3742 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3743 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3744 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3745 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3750 ip = (struct ip *)(mp->m_data + ehdrlen);
3751 if (ip->ip_p != IPPROTO_TCP)
3754 ip_hlen = ip->ip_hl << 2;
3755 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3756 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3757 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3758 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3759 /* Tell transmit desc to also do IPv4 checksum. */
3760 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3764 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3765 __func__, ntohs(eh_type));
3769 ctxd = txr->next_avail_desc;
3770 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3772 tcp_hlen = th->th_off << 2;
3774 /* This is used in the transmit desc in encap */
3775 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3777 /* VLAN MACLEN IPLEN */
3778 if (mp->m_flags & M_VLANTAG) {
3779 vtag = htole16(mp->m_pkthdr.ether_vtag);
3780 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3783 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3784 vlan_macip_lens |= ip_hlen;
3785 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3787 /* ADV DTYPE TUCMD */
3788 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3789 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3790 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3793 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3794 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3795 /* 82575 needs the queue index added */
3796 if (adapter->hw.mac.type == e1000_82575)
3797 mss_l4len_idx |= txr->me << 4;
3798 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3800 TXD->seqnum_seed = htole32(0);
3802 if (++ctxd == txr->num_desc)
3806 txr->next_avail_desc = ctxd;
3807 *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3808 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3809 *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
3814 /*********************************************************************
3816 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3818 **********************************************************************/
3821 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3822 u32 *cmd_type_len, u32 *olinfo_status)
3824 struct e1000_adv_tx_context_desc *TXD;
3825 struct adapter *adapter = txr->adapter;
3826 struct ether_vlan_header *eh;
3828 struct ip6_hdr *ip6;
3829 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
3830 int ehdrlen, ip_hlen = 0;
3834 int ctxd = txr->next_avail_desc;
3837 /* First check if TSO is to be used */
3838 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3839 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3841 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3844 /* Indicate the whole packet as payload when not doing TSO */
3845 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
3847 /* Now ready a context descriptor */
3848 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3851 ** In advanced descriptors the vlan tag must
3852 ** be placed into the context descriptor. Hence
3853 ** we need to make one even if not doing offloads.
3855 if (mp->m_flags & M_VLANTAG) {
3856 vtag = htole16(mp->m_pkthdr.ether_vtag);
3857 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3858 } else if (offload == FALSE) /* ... no offload to do */
3862 * Determine where frame payload starts.
3863 * Jump over vlan headers if already present,
3864 * helpful for QinQ too.
3866 eh = mtod(mp, struct ether_vlan_header *);
3867 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3868 etype = ntohs(eh->evl_proto);
3869 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3871 etype = ntohs(eh->evl_encap_proto);
3872 ehdrlen = ETHER_HDR_LEN;
3875 /* Set the ether header length */
3876 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3880 ip = (struct ip *)(mp->m_data + ehdrlen);
3881 ip_hlen = ip->ip_hl << 2;
3883 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3885 case ETHERTYPE_IPV6:
3886 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3887 ip_hlen = sizeof(struct ip6_hdr);
3888 /* XXX-BZ this will go badly in case of ext hdrs. */
3889 ipproto = ip6->ip6_nxt;
3890 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3897 vlan_macip_lens |= ip_hlen;
3898 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3902 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3903 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3906 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3907 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3910 #if __FreeBSD_version >= 800000
3912 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3913 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3921 if (offload) /* For the TX descriptor setup */
3922 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3924 /* 82575 needs the queue index added */
3925 if (adapter->hw.mac.type == e1000_82575)
3926 mss_l4len_idx = txr->me << 4;
3928 /* Now copy bits into descriptor */
3929 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3930 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3931 TXD->seqnum_seed = htole32(0);
3932 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3934 /* We've consumed the first desc, adjust counters */
3935 if (++ctxd == txr->num_desc)
3937 txr->next_avail_desc = ctxd;
3943 /**********************************************************************
3945 * Examine each tx_buffer in the used queue. If the hardware is done
3946 * processing the packet then free associated resources. The
3947 * tx_buffer is put back on the free queue.
3949 * TRUE return means there's work in the ring to clean, FALSE its empty.
3950 **********************************************************************/
3952 igb_txeof(struct tx_ring *txr)
3954 struct adapter *adapter = txr->adapter;
3955 struct ifnet *ifp = adapter->ifp;
3956 u32 work, processed = 0;
3957 u16 limit = txr->process_limit;
3958 struct igb_tx_buf *buf;
3959 union e1000_adv_tx_desc *txd;
3961 mtx_assert(&txr->tx_mtx, MA_OWNED);
3964 if (netmap_tx_irq(ifp, txr->me |
3965 (NETMAP_LOCKED_ENTER|NETMAP_LOCKED_EXIT)))
3967 #endif /* DEV_NETMAP */
3969 if (txr->tx_avail == txr->num_desc) {
3970 txr->queue_status = IGB_QUEUE_IDLE;
3974 /* Get work starting point */
3975 work = txr->next_to_clean;
3976 buf = &txr->tx_buffers[work];
3977 txd = &txr->tx_base[work];
3978 work -= txr->num_desc; /* The distance to ring end */
3979 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3980 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3982 union e1000_adv_tx_desc *eop = buf->eop;
3983 if (eop == NULL) /* No work */
3986 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
3987 break; /* I/O not complete */
3991 buf->m_head->m_pkthdr.len;
3992 bus_dmamap_sync(txr->txtag,
3994 BUS_DMASYNC_POSTWRITE);
3995 bus_dmamap_unload(txr->txtag,
3997 m_freem(buf->m_head);
4003 /* We clean the range if multi segment */
4004 while (txd != eop) {
4008 /* wrap the ring? */
4009 if (__predict_false(!work)) {
4010 work -= txr->num_desc;
4011 buf = txr->tx_buffers;
4016 buf->m_head->m_pkthdr.len;
4017 bus_dmamap_sync(txr->txtag,
4019 BUS_DMASYNC_POSTWRITE);
4020 bus_dmamap_unload(txr->txtag,
4022 m_freem(buf->m_head);
4032 txr->watchdog_time = ticks;
4034 /* Try the next packet */
4038 /* reset with a wrap */
4039 if (__predict_false(!work)) {
4040 work -= txr->num_desc;
4041 buf = txr->tx_buffers;
4045 } while (__predict_true(--limit));
4047 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4048 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4050 work += txr->num_desc;
4051 txr->next_to_clean = work;
4054 ** Watchdog calculation, we know there's
4055 ** work outstanding or the first return
4056 ** would have been taken, so none processed
4057 ** for too long indicates a hang.
4059 if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
4060 txr->queue_status |= IGB_QUEUE_HUNG;
4062 if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
4063 txr->queue_status &= ~IGB_QUEUE_DEPLETED;
4065 if (txr->tx_avail == txr->num_desc) {
4066 txr->queue_status = IGB_QUEUE_IDLE;
4073 /*********************************************************************
4075 * Refresh mbuf buffers for RX descriptor rings
4076 * - now keeps its own state so discards due to resource
4077 * exhaustion are unnecessary, if an mbuf cannot be obtained
4078 * it just returns, keeping its placeholder, thus it can simply
4079 * be recalled to try again.
4081 **********************************************************************/
4083 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
4085 struct adapter *adapter = rxr->adapter;
4086 bus_dma_segment_t hseg[1];
4087 bus_dma_segment_t pseg[1];
4088 struct igb_rx_buf *rxbuf;
4089 struct mbuf *mh, *mp;
4090 int i, j, nsegs, error;
4091 bool refreshed = FALSE;
4093 i = j = rxr->next_to_refresh;
4095 ** Get one descriptor beyond
4096 ** our work mark to control
4099 if (++j == adapter->num_rx_desc)
4102 while (j != limit) {
4103 rxbuf = &rxr->rx_buffers[i];
4104 /* No hdr mbuf used with header split off */
4105 if (rxr->hdr_split == FALSE)
4107 if (rxbuf->m_head == NULL) {
4108 mh = m_gethdr(M_NOWAIT, MT_DATA);
4114 mh->m_pkthdr.len = mh->m_len = MHLEN;
4116 mh->m_flags |= M_PKTHDR;
4117 /* Get the memory mapping */
4118 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4119 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
4121 printf("Refresh mbufs: hdr dmamap load"
4122 " failure - %d\n", error);
4124 rxbuf->m_head = NULL;
4128 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4129 BUS_DMASYNC_PREREAD);
4130 rxr->rx_base[i].read.hdr_addr =
4131 htole64(hseg[0].ds_addr);
4133 if (rxbuf->m_pack == NULL) {
4134 mp = m_getjcl(M_NOWAIT, MT_DATA,
4135 M_PKTHDR, adapter->rx_mbuf_sz);
4141 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4142 /* Get the memory mapping */
4143 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4144 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
4146 printf("Refresh mbufs: payload dmamap load"
4147 " failure - %d\n", error);
4149 rxbuf->m_pack = NULL;
4153 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4154 BUS_DMASYNC_PREREAD);
4155 rxr->rx_base[i].read.pkt_addr =
4156 htole64(pseg[0].ds_addr);
4157 refreshed = TRUE; /* I feel wefreshed :) */
4159 i = j; /* our next is precalculated */
4160 rxr->next_to_refresh = i;
4161 if (++j == adapter->num_rx_desc)
4165 if (refreshed) /* update tail */
4166 E1000_WRITE_REG(&adapter->hw,
4167 E1000_RDT(rxr->me), rxr->next_to_refresh);
4172 /*********************************************************************
4174 * Allocate memory for rx_buffer structures. Since we use one
4175 * rx_buffer per received packet, the maximum number of rx_buffer's
4176 * that we'll need is equal to the number of receive descriptors
4177 * that we've allocated.
4179 **********************************************************************/
4181 igb_allocate_receive_buffers(struct rx_ring *rxr)
4183 struct adapter *adapter = rxr->adapter;
4184 device_t dev = adapter->dev;
4185 struct igb_rx_buf *rxbuf;
4186 int i, bsize, error;
4188 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
4189 if (!(rxr->rx_buffers =
4190 (struct igb_rx_buf *) malloc(bsize,
4191 M_DEVBUF, M_NOWAIT | M_ZERO))) {
4192 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4197 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4198 1, 0, /* alignment, bounds */
4199 BUS_SPACE_MAXADDR, /* lowaddr */
4200 BUS_SPACE_MAXADDR, /* highaddr */
4201 NULL, NULL, /* filter, filterarg */
4202 MSIZE, /* maxsize */
4204 MSIZE, /* maxsegsize */
4206 NULL, /* lockfunc */
4207 NULL, /* lockfuncarg */
4209 device_printf(dev, "Unable to create RX DMA tag\n");
4213 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4214 1, 0, /* alignment, bounds */
4215 BUS_SPACE_MAXADDR, /* lowaddr */
4216 BUS_SPACE_MAXADDR, /* highaddr */
4217 NULL, NULL, /* filter, filterarg */
4218 MJUM9BYTES, /* maxsize */
4220 MJUM9BYTES, /* maxsegsize */
4222 NULL, /* lockfunc */
4223 NULL, /* lockfuncarg */
4225 device_printf(dev, "Unable to create RX payload DMA tag\n");
4229 for (i = 0; i < adapter->num_rx_desc; i++) {
4230 rxbuf = &rxr->rx_buffers[i];
4231 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
4234 "Unable to create RX head DMA maps\n");
4237 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
4240 "Unable to create RX packet DMA maps\n");
4248 /* Frees all, but can handle partial completion */
4249 igb_free_receive_structures(adapter);
4255 igb_free_receive_ring(struct rx_ring *rxr)
4257 struct adapter *adapter = rxr->adapter;
4258 struct igb_rx_buf *rxbuf;
4261 for (int i = 0; i < adapter->num_rx_desc; i++) {
4262 rxbuf = &rxr->rx_buffers[i];
4263 if (rxbuf->m_head != NULL) {
4264 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4265 BUS_DMASYNC_POSTREAD);
4266 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4267 rxbuf->m_head->m_flags |= M_PKTHDR;
4268 m_freem(rxbuf->m_head);
4270 if (rxbuf->m_pack != NULL) {
4271 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4272 BUS_DMASYNC_POSTREAD);
4273 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4274 rxbuf->m_pack->m_flags |= M_PKTHDR;
4275 m_freem(rxbuf->m_pack);
4277 rxbuf->m_head = NULL;
4278 rxbuf->m_pack = NULL;
4283 /*********************************************************************
4285 * Initialize a receive ring and its buffers.
4287 **********************************************************************/
4289 igb_setup_receive_ring(struct rx_ring *rxr)
4291 struct adapter *adapter;
4294 struct igb_rx_buf *rxbuf;
4295 bus_dma_segment_t pseg[1], hseg[1];
4296 struct lro_ctrl *lro = &rxr->lro;
4297 int rsize, nsegs, error = 0;
4299 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4300 struct netmap_slot *slot;
4301 #endif /* DEV_NETMAP */
4303 adapter = rxr->adapter;
4307 /* Clear the ring contents */
4310 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4311 #endif /* DEV_NETMAP */
4312 rsize = roundup2(adapter->num_rx_desc *
4313 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
4314 bzero((void *)rxr->rx_base, rsize);
4317 ** Free current RX buffer structures and their mbufs
4319 igb_free_receive_ring(rxr);
4321 /* Configure for header split? */
4322 if (igb_header_split)
4323 rxr->hdr_split = TRUE;
4325 /* Now replenish the ring mbufs */
4326 for (int j = 0; j < adapter->num_rx_desc; ++j) {
4327 struct mbuf *mh, *mp;
4329 rxbuf = &rxr->rx_buffers[j];
4332 /* slot sj is mapped to the i-th NIC-ring entry */
4333 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4337 addr = PNMB(slot + sj, &paddr);
4338 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4339 /* Update descriptor */
4340 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4343 #endif /* DEV_NETMAP */
4344 if (rxr->hdr_split == FALSE)
4347 /* First the header */
4348 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4349 if (rxbuf->m_head == NULL) {
4353 m_adj(rxbuf->m_head, ETHER_ALIGN);
4355 mh->m_len = mh->m_pkthdr.len = MHLEN;
4356 mh->m_flags |= M_PKTHDR;
4357 /* Get the memory mapping */
4358 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4359 rxbuf->hmap, rxbuf->m_head, hseg,
4360 &nsegs, BUS_DMA_NOWAIT);
4361 if (error != 0) /* Nothing elegant to do here */
4363 bus_dmamap_sync(rxr->htag,
4364 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4365 /* Update descriptor */
4366 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4369 /* Now the payload cluster */
4370 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4371 M_PKTHDR, adapter->rx_mbuf_sz);
4372 if (rxbuf->m_pack == NULL) {
4377 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4378 /* Get the memory mapping */
4379 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4380 rxbuf->pmap, mp, pseg,
4381 &nsegs, BUS_DMA_NOWAIT);
4384 bus_dmamap_sync(rxr->ptag,
4385 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4386 /* Update descriptor */
4387 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4390 /* Setup our descriptor indices */
4391 rxr->next_to_check = 0;
4392 rxr->next_to_refresh = adapter->num_rx_desc - 1;
4393 rxr->lro_enabled = FALSE;
4394 rxr->rx_split_packets = 0;
4399 rxr->discard = FALSE;
4401 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4402 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4405 ** Now set up the LRO interface, we
4406 ** also only do head split when LRO
4407 ** is enabled, since so often they
4408 ** are undesireable in similar setups.
4410 if (ifp->if_capenable & IFCAP_LRO) {
4411 error = tcp_lro_init(lro);
4413 device_printf(dev, "LRO Initialization failed!\n");
4416 INIT_DEBUGOUT("RX LRO Initialized\n");
4417 rxr->lro_enabled = TRUE;
4418 lro->ifp = adapter->ifp;
4425 igb_free_receive_ring(rxr);
4431 /*********************************************************************
4433 * Initialize all receive rings.
4435 **********************************************************************/
4437 igb_setup_receive_structures(struct adapter *adapter)
4439 struct rx_ring *rxr = adapter->rx_rings;
4442 for (i = 0; i < adapter->num_queues; i++, rxr++)
4443 if (igb_setup_receive_ring(rxr))
4449 * Free RX buffers allocated so far, we will only handle
4450 * the rings that completed, the failing case will have
4451 * cleaned up for itself. 'i' is the endpoint.
4453 for (int j = 0; j < i; ++j) {
4454 rxr = &adapter->rx_rings[j];
4456 igb_free_receive_ring(rxr);
4463 /*********************************************************************
4465 * Enable receive unit.
4467 **********************************************************************/
4469 igb_initialize_receive_units(struct adapter *adapter)
4471 struct rx_ring *rxr = adapter->rx_rings;
4472 struct ifnet *ifp = adapter->ifp;
4473 struct e1000_hw *hw = &adapter->hw;
4474 u32 rctl, rxcsum, psize, srrctl = 0;
4476 INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
4479 * Make sure receives are disabled while setting
4480 * up the descriptor ring
4482 rctl = E1000_READ_REG(hw, E1000_RCTL);
4483 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4486 ** Set up for header split
4488 if (igb_header_split) {
4489 /* Use a standard mbuf for the header */
4490 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4491 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4493 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4496 ** Set up for jumbo frames
4498 if (ifp->if_mtu > ETHERMTU) {
4499 rctl |= E1000_RCTL_LPE;
4500 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
4501 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4502 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4503 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
4504 srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4505 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4507 /* Set maximum packet len */
4508 psize = adapter->max_frame_size;
4509 /* are we on a vlan? */
4510 if (adapter->ifp->if_vlantrunk != NULL)
4511 psize += VLAN_TAG_SIZE;
4512 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
4514 rctl &= ~E1000_RCTL_LPE;
4515 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4516 rctl |= E1000_RCTL_SZ_2048;
4519 /* Setup the Base and Length of the Rx Descriptor Rings */
4520 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4521 u64 bus_addr = rxr->rxdma.dma_paddr;
4524 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4525 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4526 E1000_WRITE_REG(hw, E1000_RDBAH(i),
4527 (uint32_t)(bus_addr >> 32));
4528 E1000_WRITE_REG(hw, E1000_RDBAL(i),
4529 (uint32_t)bus_addr);
4530 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
4531 /* Enable this Queue */
4532 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
4533 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4534 rxdctl &= 0xFFF00000;
4535 rxdctl |= IGB_RX_PTHRESH;
4536 rxdctl |= IGB_RX_HTHRESH << 8;
4537 rxdctl |= IGB_RX_WTHRESH << 16;
4538 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
4542 ** Setup for RX MultiQueue
4544 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4545 if (adapter->num_queues >1) {
4546 u32 random[10], mrqc, shift = 0;
4552 arc4rand(&random, sizeof(random), 0);
4553 if (adapter->hw.mac.type == e1000_82575)
4555 /* Warning FM follows */
4556 for (int i = 0; i < 128; i++) {
4558 (i % adapter->num_queues) << shift;
4561 E1000_RETA(i >> 2), reta.dword);
4563 /* Now fill in hash table */
4564 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
4565 for (int i = 0; i < 10; i++)
4566 E1000_WRITE_REG_ARRAY(hw,
4567 E1000_RSSRK(0), i, random[i]);
4569 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
4570 E1000_MRQC_RSS_FIELD_IPV4_TCP);
4571 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
4572 E1000_MRQC_RSS_FIELD_IPV6_TCP);
4573 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
4574 E1000_MRQC_RSS_FIELD_IPV6_UDP);
4575 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
4576 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
4578 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
4581 ** NOTE: Receive Full-Packet Checksum Offload
4582 ** is mutually exclusive with Multiqueue. However
4583 ** this is not the same as TCP/IP checksums which
4586 rxcsum |= E1000_RXCSUM_PCSD;
4587 #if __FreeBSD_version >= 800000
4588 /* For SCTP Offload */
4589 if ((hw->mac.type == e1000_82576)
4590 && (ifp->if_capenable & IFCAP_RXCSUM))
4591 rxcsum |= E1000_RXCSUM_CRCOFL;
4595 if (ifp->if_capenable & IFCAP_RXCSUM) {
4596 rxcsum |= E1000_RXCSUM_IPPCSE;
4597 #if __FreeBSD_version >= 800000
4598 if (adapter->hw.mac.type == e1000_82576)
4599 rxcsum |= E1000_RXCSUM_CRCOFL;
4602 rxcsum &= ~E1000_RXCSUM_TUOFL;
4604 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4606 /* Setup the Receive Control Register */
4607 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4608 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4609 E1000_RCTL_RDMTS_HALF |
4610 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4611 /* Strip CRC bytes. */
4612 rctl |= E1000_RCTL_SECRC;
4613 /* Make sure VLAN Filters are off */
4614 rctl &= ~E1000_RCTL_VFE;
4615 /* Don't store bad packets */
4616 rctl &= ~E1000_RCTL_SBP;
4618 /* Enable Receives */
4619 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4622 * Setup the HW Rx Head and Tail Descriptor Pointers
4623 * - needs to be after enable
4625 for (int i = 0; i < adapter->num_queues; i++) {
4626 rxr = &adapter->rx_rings[i];
4627 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
4630 * an init() while a netmap client is active must
4631 * preserve the rx buffers passed to userspace.
4632 * In this driver it means we adjust RDT to
4633 * somthing different from next_to_refresh
4634 * (which is not used in netmap mode).
4636 if (ifp->if_capenable & IFCAP_NETMAP) {
4637 struct netmap_adapter *na = NA(adapter->ifp);
4638 struct netmap_kring *kring = &na->rx_rings[i];
4639 int t = rxr->next_to_refresh - kring->nr_hwavail;
4641 if (t >= adapter->num_rx_desc)
4642 t -= adapter->num_rx_desc;
4644 t += adapter->num_rx_desc;
4645 E1000_WRITE_REG(hw, E1000_RDT(i), t);
4647 #endif /* DEV_NETMAP */
4648 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
4653 /*********************************************************************
4655 * Free receive rings.
4657 **********************************************************************/
4659 igb_free_receive_structures(struct adapter *adapter)
4661 struct rx_ring *rxr = adapter->rx_rings;
4663 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4664 struct lro_ctrl *lro = &rxr->lro;
4665 igb_free_receive_buffers(rxr);
4667 igb_dma_free(adapter, &rxr->rxdma);
4670 free(adapter->rx_rings, M_DEVBUF);
4673 /*********************************************************************
4675 * Free receive ring data structures.
4677 **********************************************************************/
4679 igb_free_receive_buffers(struct rx_ring *rxr)
4681 struct adapter *adapter = rxr->adapter;
4682 struct igb_rx_buf *rxbuf;
4685 INIT_DEBUGOUT("free_receive_structures: begin");
4687 /* Cleanup any existing buffers */
4688 if (rxr->rx_buffers != NULL) {
4689 for (i = 0; i < adapter->num_rx_desc; i++) {
4690 rxbuf = &rxr->rx_buffers[i];
4691 if (rxbuf->m_head != NULL) {
4692 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4693 BUS_DMASYNC_POSTREAD);
4694 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4695 rxbuf->m_head->m_flags |= M_PKTHDR;
4696 m_freem(rxbuf->m_head);
4698 if (rxbuf->m_pack != NULL) {
4699 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4700 BUS_DMASYNC_POSTREAD);
4701 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4702 rxbuf->m_pack->m_flags |= M_PKTHDR;
4703 m_freem(rxbuf->m_pack);
4705 rxbuf->m_head = NULL;
4706 rxbuf->m_pack = NULL;
4707 if (rxbuf->hmap != NULL) {
4708 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4711 if (rxbuf->pmap != NULL) {
4712 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4716 if (rxr->rx_buffers != NULL) {
4717 free(rxr->rx_buffers, M_DEVBUF);
4718 rxr->rx_buffers = NULL;
4722 if (rxr->htag != NULL) {
4723 bus_dma_tag_destroy(rxr->htag);
4726 if (rxr->ptag != NULL) {
4727 bus_dma_tag_destroy(rxr->ptag);
4732 static __inline void
4733 igb_rx_discard(struct rx_ring *rxr, int i)
4735 struct igb_rx_buf *rbuf;
4737 rbuf = &rxr->rx_buffers[i];
4739 /* Partially received? Free the chain */
4740 if (rxr->fmp != NULL) {
4741 rxr->fmp->m_flags |= M_PKTHDR;
4748 ** With advanced descriptors the writeback
4749 ** clobbers the buffer addrs, so its easier
4750 ** to just free the existing mbufs and take
4751 ** the normal refresh path to get new buffers
4755 m_free(rbuf->m_head);
4756 rbuf->m_head = NULL;
4757 bus_dmamap_unload(rxr->htag, rbuf->hmap);
4761 m_free(rbuf->m_pack);
4762 rbuf->m_pack = NULL;
4763 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
4769 static __inline void
4770 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4774 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4775 * should be computed by hardware. Also it should not have VLAN tag in
4778 if (rxr->lro_enabled &&
4779 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4780 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4781 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4782 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4783 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4784 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4786 * Send to the stack if:
4787 ** - LRO not enabled, or
4788 ** - no LRO resources, or
4789 ** - lro enqueue fails
4791 if (rxr->lro.lro_cnt != 0)
4792 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4796 (*ifp->if_input)(ifp, m);
4800 /*********************************************************************
4802 * This routine executes in interrupt context. It replenishes
4803 * the mbufs in the descriptor and sends data which has been
4804 * dma'ed into host memory to upper layer.
4806 * We loop at most count times if count is > 0, or until done if
4809 * Return TRUE if more to clean, FALSE otherwise
4810 *********************************************************************/
4812 igb_rxeof(struct igb_queue *que, int count, int *done)
4814 struct adapter *adapter = que->adapter;
4815 struct rx_ring *rxr = que->rxr;
4816 struct ifnet *ifp = adapter->ifp;
4817 struct lro_ctrl *lro = &rxr->lro;
4818 struct lro_entry *queued;
4819 int i, processed = 0, rxdone = 0;
4820 u32 ptype, staterr = 0;
4821 union e1000_adv_rx_desc *cur;
4824 /* Sync the ring. */
4825 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4826 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4829 if (netmap_rx_irq(ifp, rxr->me | NETMAP_LOCKED_ENTER, &processed))
4831 #endif /* DEV_NETMAP */
4833 /* Main clean loop */
4834 for (i = rxr->next_to_check; count != 0;) {
4835 struct mbuf *sendmp, *mh, *mp;
4836 struct igb_rx_buf *rxbuf;
4837 u16 hlen, plen, hdr, vtag;
4840 cur = &rxr->rx_base[i];
4841 staterr = le32toh(cur->wb.upper.status_error);
4842 if ((staterr & E1000_RXD_STAT_DD) == 0)
4844 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4847 sendmp = mh = mp = NULL;
4848 cur->wb.upper.status_error = 0;
4849 rxbuf = &rxr->rx_buffers[i];
4850 plen = le16toh(cur->wb.upper.length);
4851 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
4852 if (((adapter->hw.mac.type == e1000_i350) ||
4853 (adapter->hw.mac.type == e1000_i354)) &&
4854 (staterr & E1000_RXDEXT_STATERR_LB))
4855 vtag = be16toh(cur->wb.upper.vlan);
4857 vtag = le16toh(cur->wb.upper.vlan);
4858 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4859 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
4861 /* Make sure all segments of a bad packet are discarded */
4862 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) ||
4864 adapter->dropped_pkts++;
4865 ++rxr->rx_discarded;
4866 if (!eop) /* Catch subsequent segs */
4867 rxr->discard = TRUE;
4869 rxr->discard = FALSE;
4870 igb_rx_discard(rxr, i);
4875 ** The way the hardware is configured to
4876 ** split, it will ONLY use the header buffer
4877 ** when header split is enabled, otherwise we
4878 ** get normal behavior, ie, both header and
4879 ** payload are DMA'd into the payload buffer.
4881 ** The fmp test is to catch the case where a
4882 ** packet spans multiple descriptors, in that
4883 ** case only the first header is valid.
4885 if (rxr->hdr_split && rxr->fmp == NULL) {
4886 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4887 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4888 E1000_RXDADV_HDRBUFLEN_SHIFT;
4889 if (hlen > IGB_HDR_BUF)
4891 mh = rxr->rx_buffers[i].m_head;
4893 /* clear buf pointer for refresh */
4894 rxbuf->m_head = NULL;
4896 ** Get the payload length, this
4897 ** could be zero if its a small
4901 mp = rxr->rx_buffers[i].m_pack;
4904 /* clear buf pointer */
4905 rxbuf->m_pack = NULL;
4906 rxr->rx_split_packets++;
4910 ** Either no header split, or a
4911 ** secondary piece of a fragmented
4914 mh = rxr->rx_buffers[i].m_pack;
4916 /* clear buf info for refresh */
4917 rxbuf->m_pack = NULL;
4919 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4921 ++processed; /* So we know when to refresh */
4923 /* Initial frame - setup */
4924 if (rxr->fmp == NULL) {
4925 mh->m_pkthdr.len = mh->m_len;
4926 /* Save the head of the chain */
4930 /* Add payload if split */
4931 mh->m_pkthdr.len += mp->m_len;
4932 rxr->lmp = mh->m_next;
4935 /* Chain mbuf's together */
4936 rxr->lmp->m_next = mh;
4937 rxr->lmp = rxr->lmp->m_next;
4938 rxr->fmp->m_pkthdr.len += mh->m_len;
4942 rxr->fmp->m_pkthdr.rcvif = ifp;
4945 /* capture data for AIM */
4947 rxr->bytes += rxr->fmp->m_pkthdr.len;
4948 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4950 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4951 igb_rx_checksum(staterr, rxr->fmp, ptype);
4953 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4954 (staterr & E1000_RXD_STAT_VP) != 0) {
4955 rxr->fmp->m_pkthdr.ether_vtag = vtag;
4956 rxr->fmp->m_flags |= M_VLANTAG;
4958 #ifndef IGB_LEGACY_TX
4959 rxr->fmp->m_pkthdr.flowid = que->msix;
4960 rxr->fmp->m_flags |= M_FLOWID;
4963 /* Make sure to set M_PKTHDR. */
4964 sendmp->m_flags |= M_PKTHDR;
4970 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4971 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4973 /* Advance our pointers to the next descriptor. */
4974 if (++i == adapter->num_rx_desc)
4977 ** Send to the stack or LRO
4979 if (sendmp != NULL) {
4980 rxr->next_to_check = i;
4981 igb_rx_input(rxr, ifp, sendmp, ptype);
4982 i = rxr->next_to_check;
4986 /* Every 8 descriptors we go to refresh mbufs */
4987 if (processed == 8) {
4988 igb_refresh_mbufs(rxr, i);
4993 /* Catch any remainders */
4994 if (igb_rx_unrefreshed(rxr))
4995 igb_refresh_mbufs(rxr, i);
4997 rxr->next_to_check = i;
5000 * Flush any outstanding LRO work
5002 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
5003 SLIST_REMOVE_HEAD(&lro->lro_active, next);
5004 tcp_lro_flush(lro, queued);
5011 return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
5014 /*********************************************************************
5016 * Verify that the hardware indicated that the checksum is valid.
5017 * Inform the stack about the status of checksum so that stack
5018 * doesn't spend time verifying the checksum.
5020 *********************************************************************/
5022 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
5024 u16 status = (u16)staterr;
5025 u8 errors = (u8) (staterr >> 24);
5028 /* Ignore Checksum bit is set */
5029 if (status & E1000_RXD_STAT_IXSM) {
5030 mp->m_pkthdr.csum_flags = 0;
5034 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
5035 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
5039 if (status & E1000_RXD_STAT_IPCS) {
5041 if (!(errors & E1000_RXD_ERR_IPE)) {
5042 /* IP Checksum Good */
5043 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
5044 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
5046 mp->m_pkthdr.csum_flags = 0;
5049 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
5050 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5051 #if __FreeBSD_version >= 800000
5052 if (sctp) /* reassign */
5053 type = CSUM_SCTP_VALID;
5056 if (!(errors & E1000_RXD_ERR_TCPE)) {
5057 mp->m_pkthdr.csum_flags |= type;
5059 mp->m_pkthdr.csum_data = htons(0xffff);
5066 * This routine is run via an vlan
5070 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5072 struct adapter *adapter = ifp->if_softc;
5075 if (ifp->if_softc != arg) /* Not our event */
5078 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5081 IGB_CORE_LOCK(adapter);
5082 index = (vtag >> 5) & 0x7F;
5084 adapter->shadow_vfta[index] |= (1 << bit);
5085 ++adapter->num_vlans;
5086 /* Change hw filter setting */
5087 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5088 igb_setup_vlan_hw_support(adapter);
5089 IGB_CORE_UNLOCK(adapter);
5093 * This routine is run via an vlan
5097 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5099 struct adapter *adapter = ifp->if_softc;
5102 if (ifp->if_softc != arg)
5105 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5108 IGB_CORE_LOCK(adapter);
5109 index = (vtag >> 5) & 0x7F;
5111 adapter->shadow_vfta[index] &= ~(1 << bit);
5112 --adapter->num_vlans;
5113 /* Change hw filter setting */
5114 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5115 igb_setup_vlan_hw_support(adapter);
5116 IGB_CORE_UNLOCK(adapter);
5120 igb_setup_vlan_hw_support(struct adapter *adapter)
5122 struct e1000_hw *hw = &adapter->hw;
5123 struct ifnet *ifp = adapter->ifp;
5126 if (adapter->vf_ifp) {
5127 e1000_rlpml_set_vf(hw,
5128 adapter->max_frame_size + VLAN_TAG_SIZE);
5132 reg = E1000_READ_REG(hw, E1000_CTRL);
5133 reg |= E1000_CTRL_VME;
5134 E1000_WRITE_REG(hw, E1000_CTRL, reg);
5136 /* Enable the Filter Table */
5137 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
5138 reg = E1000_READ_REG(hw, E1000_RCTL);
5139 reg &= ~E1000_RCTL_CFIEN;
5140 reg |= E1000_RCTL_VFE;
5141 E1000_WRITE_REG(hw, E1000_RCTL, reg);
5144 /* Update the frame size */
5145 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
5146 adapter->max_frame_size + VLAN_TAG_SIZE);
5148 /* Don't bother with table if no vlans */
5149 if ((adapter->num_vlans == 0) ||
5150 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
5153 ** A soft reset zero's out the VFTA, so
5154 ** we need to repopulate it now.
5156 for (int i = 0; i < IGB_VFTA_SIZE; i++)
5157 if (adapter->shadow_vfta[i] != 0) {
5158 if (adapter->vf_ifp)
5159 e1000_vfta_set_vf(hw,
5160 adapter->shadow_vfta[i], TRUE);
5162 e1000_write_vfta(hw,
5163 i, adapter->shadow_vfta[i]);
5168 igb_enable_intr(struct adapter *adapter)
5170 /* With RSS set up what to auto clear */
5171 if (adapter->msix_mem) {
5172 u32 mask = (adapter->que_mask | adapter->link_mask);
5173 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
5174 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
5175 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
5176 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5179 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5182 E1000_WRITE_FLUSH(&adapter->hw);
5188 igb_disable_intr(struct adapter *adapter)
5190 if (adapter->msix_mem) {
5191 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
5192 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
5194 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
5195 E1000_WRITE_FLUSH(&adapter->hw);
5200 * Bit of a misnomer, what this really means is
5201 * to enable OS management of the system... aka
5202 * to disable special hardware management features
5205 igb_init_manageability(struct adapter *adapter)
5207 if (adapter->has_manage) {
5208 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
5209 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5211 /* disable hardware interception of ARP */
5212 manc &= ~(E1000_MANC_ARP_EN);
5214 /* enable receiving management packets to the host */
5215 manc |= E1000_MANC_EN_MNG2HOST;
5216 manc2h |= 1 << 5; /* Mng Port 623 */
5217 manc2h |= 1 << 6; /* Mng Port 664 */
5218 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
5219 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5224 * Give control back to hardware management
5225 * controller if there is one.
5228 igb_release_manageability(struct adapter *adapter)
5230 if (adapter->has_manage) {
5231 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5233 /* re-enable hardware interception of ARP */
5234 manc |= E1000_MANC_ARP_EN;
5235 manc &= ~E1000_MANC_EN_MNG2HOST;
5237 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5242 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
5243 * For ASF and Pass Through versions of f/w this means that
5244 * the driver is loaded.
5248 igb_get_hw_control(struct adapter *adapter)
5252 if (adapter->vf_ifp)
5255 /* Let firmware know the driver has taken over */
5256 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5257 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5258 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
5262 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
5263 * For ASF and Pass Through versions of f/w this means that the
5264 * driver is no longer loaded.
5268 igb_release_hw_control(struct adapter *adapter)
5272 if (adapter->vf_ifp)
5275 /* Let firmware taken over control of h/w */
5276 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5277 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5278 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
5282 igb_is_valid_ether_addr(uint8_t *addr)
5284 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
5286 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
5295 * Enable PCI Wake On Lan capability
5298 igb_enable_wakeup(device_t dev)
5303 /* First find the capabilities pointer*/
5304 cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
5305 /* Read the PM Capabilities */
5306 id = pci_read_config(dev, cap, 1);
5307 if (id != PCIY_PMG) /* Something wrong */
5309 /* OK, we have the power capabilities, so
5310 now get the status register */
5311 cap += PCIR_POWER_STATUS;
5312 status = pci_read_config(dev, cap, 2);
5313 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5314 pci_write_config(dev, cap, status, 2);
5319 igb_led_func(void *arg, int onoff)
5321 struct adapter *adapter = arg;
5323 IGB_CORE_LOCK(adapter);
5325 e1000_setup_led(&adapter->hw);
5326 e1000_led_on(&adapter->hw);
5328 e1000_led_off(&adapter->hw);
5329 e1000_cleanup_led(&adapter->hw);
5331 IGB_CORE_UNLOCK(adapter);
5334 /**********************************************************************
5336 * Update the board statistics counters.
5338 **********************************************************************/
5340 igb_update_stats_counters(struct adapter *adapter)
5343 struct e1000_hw *hw = &adapter->hw;
5344 struct e1000_hw_stats *stats;
5347 ** The virtual function adapter has only a
5348 ** small controlled set of stats, do only
5349 ** those and return.
5351 if (adapter->vf_ifp) {
5352 igb_update_vf_stats_counters(adapter);
5356 stats = (struct e1000_hw_stats *)adapter->stats;
5358 if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5359 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
5361 E1000_READ_REG(hw,E1000_SYMERRS);
5362 stats->sec += E1000_READ_REG(hw, E1000_SEC);
5365 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5366 stats->mpc += E1000_READ_REG(hw, E1000_MPC);
5367 stats->scc += E1000_READ_REG(hw, E1000_SCC);
5368 stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
5370 stats->mcc += E1000_READ_REG(hw, E1000_MCC);
5371 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
5372 stats->colc += E1000_READ_REG(hw, E1000_COLC);
5373 stats->dc += E1000_READ_REG(hw, E1000_DC);
5374 stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
5375 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5376 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5378 ** For watchdog management we need to know if we have been
5379 ** paused during the last interval, so capture that here.
5381 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5382 stats->xoffrxc += adapter->pause_frames;
5383 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5384 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5385 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
5386 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
5387 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
5388 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
5389 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5390 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5391 stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
5392 stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
5393 stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
5394 stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
5396 /* For the 64-bit byte counters the low dword must be read first. */
5397 /* Both registers clear on the read of the high dword */
5399 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
5400 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
5401 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
5402 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
5404 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
5405 stats->ruc += E1000_READ_REG(hw, E1000_RUC);
5406 stats->rfc += E1000_READ_REG(hw, E1000_RFC);
5407 stats->roc += E1000_READ_REG(hw, E1000_ROC);
5408 stats->rjc += E1000_READ_REG(hw, E1000_RJC);
5410 stats->tor += E1000_READ_REG(hw, E1000_TORH);
5411 stats->tot += E1000_READ_REG(hw, E1000_TOTH);
5413 stats->tpr += E1000_READ_REG(hw, E1000_TPR);
5414 stats->tpt += E1000_READ_REG(hw, E1000_TPT);
5415 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5416 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5417 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5418 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5419 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5420 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5421 stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
5422 stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
5424 /* Interrupt Counts */
5426 stats->iac += E1000_READ_REG(hw, E1000_IAC);
5427 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5428 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5429 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5430 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5431 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5432 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5433 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5434 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5436 /* Host to Card Statistics */
5438 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
5439 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
5440 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
5441 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
5442 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
5443 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
5444 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
5445 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
5446 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
5447 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
5448 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
5449 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
5450 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
5451 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
5453 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5454 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5455 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5456 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
5457 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5458 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5461 ifp->if_collisions = stats->colc;
5464 ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc +
5465 stats->crcerrs + stats->algnerrc +
5466 stats->ruc + stats->roc + stats->mpc + stats->cexterr;
5469 ifp->if_oerrors = stats->ecol +
5470 stats->latecol + adapter->watchdog_events;
5472 /* Driver specific counters */
5473 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
5474 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
5475 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
5476 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
5477 adapter->packet_buf_alloc_tx =
5478 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
5479 adapter->packet_buf_alloc_rx =
5480 (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
5484 /**********************************************************************
5486 * Initialize the VF board statistics counters.
5488 **********************************************************************/
5490 igb_vf_init_stats(struct adapter *adapter)
5492 struct e1000_hw *hw = &adapter->hw;
5493 struct e1000_vf_stats *stats;
5495 stats = (struct e1000_vf_stats *)adapter->stats;
5498 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
5499 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
5500 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
5501 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
5502 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
5505 /**********************************************************************
5507 * Update the VF board statistics counters.
5509 **********************************************************************/
5511 igb_update_vf_stats_counters(struct adapter *adapter)
5513 struct e1000_hw *hw = &adapter->hw;
5514 struct e1000_vf_stats *stats;
5516 if (adapter->link_speed == 0)
5519 stats = (struct e1000_vf_stats *)adapter->stats;
5521 UPDATE_VF_REG(E1000_VFGPRC,
5522 stats->last_gprc, stats->gprc);
5523 UPDATE_VF_REG(E1000_VFGORC,
5524 stats->last_gorc, stats->gorc);
5525 UPDATE_VF_REG(E1000_VFGPTC,
5526 stats->last_gptc, stats->gptc);
5527 UPDATE_VF_REG(E1000_VFGOTC,
5528 stats->last_gotc, stats->gotc);
5529 UPDATE_VF_REG(E1000_VFMPRC,
5530 stats->last_mprc, stats->mprc);
5533 /* Export a single 32-bit register via a read-only sysctl. */
5535 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5537 struct adapter *adapter;
5540 adapter = oidp->oid_arg1;
5541 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5542 return (sysctl_handle_int(oidp, &val, 0, req));
5546 ** Tuneable interrupt rate handler
5549 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5551 struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1);
5553 u32 reg, usec, rate;
5555 reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
5556 usec = ((reg & 0x7FFC) >> 2);
5558 rate = 1000000 / usec;
5561 error = sysctl_handle_int(oidp, &rate, 0, req);
5562 if (error || !req->newptr)
5568 * Add sysctl variables, one per statistic, to the system.
5571 igb_add_hw_stats(struct adapter *adapter)
5573 device_t dev = adapter->dev;
5575 struct tx_ring *txr = adapter->tx_rings;
5576 struct rx_ring *rxr = adapter->rx_rings;
5578 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5579 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5580 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5581 struct e1000_hw_stats *stats = adapter->stats;
5583 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
5584 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
5586 #define QUEUE_NAME_LEN 32
5587 char namebuf[QUEUE_NAME_LEN];
5589 /* Driver Statistics */
5590 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "link_irq",
5591 CTLFLAG_RD, &adapter->link_irq, 0,
5592 "Link MSIX IRQ Handled");
5593 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5594 CTLFLAG_RD, &adapter->dropped_pkts,
5595 "Driver dropped packets");
5596 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5597 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5598 "Driver tx dma failure in xmit");
5599 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5600 CTLFLAG_RD, &adapter->rx_overruns,
5602 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5603 CTLFLAG_RD, &adapter->watchdog_events,
5604 "Watchdog timeouts");
5606 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
5607 CTLFLAG_RD, &adapter->device_control,
5608 "Device Control Register");
5609 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
5610 CTLFLAG_RD, &adapter->rx_control,
5611 "Receiver Control Register");
5612 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
5613 CTLFLAG_RD, &adapter->int_mask,
5615 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
5616 CTLFLAG_RD, &adapter->eint_mask,
5617 "Extended Interrupt Mask");
5618 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
5619 CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
5620 "Transmit Buffer Packet Allocation");
5621 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
5622 CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
5623 "Receive Buffer Packet Allocation");
5624 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5625 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5626 "Flow Control High Watermark");
5627 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5628 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5629 "Flow Control Low Watermark");
5631 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5632 struct lro_ctrl *lro = &rxr->lro;
5634 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5635 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5636 CTLFLAG_RD, NULL, "Queue Name");
5637 queue_list = SYSCTL_CHILDREN(queue_node);
5639 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5640 CTLFLAG_RD, &adapter->queues[i],
5641 sizeof(&adapter->queues[i]),
5642 igb_sysctl_interrupt_rate_handler,
5643 "IU", "Interrupt Rate");
5645 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5646 CTLFLAG_RD, adapter, E1000_TDH(txr->me),
5647 igb_sysctl_reg_handler, "IU",
5648 "Transmit Descriptor Head");
5649 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5650 CTLFLAG_RD, adapter, E1000_TDT(txr->me),
5651 igb_sysctl_reg_handler, "IU",
5652 "Transmit Descriptor Tail");
5653 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5654 CTLFLAG_RD, &txr->no_desc_avail,
5655 "Queue No Descriptor Available");
5656 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5657 CTLFLAG_RD, &txr->total_packets,
5658 "Queue Packets Transmitted");
5660 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5661 CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
5662 igb_sysctl_reg_handler, "IU",
5663 "Receive Descriptor Head");
5664 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5665 CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
5666 igb_sysctl_reg_handler, "IU",
5667 "Receive Descriptor Tail");
5668 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5669 CTLFLAG_RD, &rxr->rx_packets,
5670 "Queue Packets Received");
5671 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5672 CTLFLAG_RD, &rxr->rx_bytes,
5673 "Queue Bytes Received");
5674 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
5675 CTLFLAG_RD, &lro->lro_queued, 0,
5677 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
5678 CTLFLAG_RD, &lro->lro_flushed, 0,
5682 /* MAC stats get their own sub node */
5684 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5685 CTLFLAG_RD, NULL, "MAC Statistics");
5686 stat_list = SYSCTL_CHILDREN(stat_node);
5689 ** VF adapter has a very limited set of stats
5690 ** since its not managing the metal, so to speak.
5692 if (adapter->vf_ifp) {
5693 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5694 CTLFLAG_RD, &stats->gprc,
5695 "Good Packets Received");
5696 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5697 CTLFLAG_RD, &stats->gptc,
5698 "Good Packets Transmitted");
5699 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5700 CTLFLAG_RD, &stats->gorc,
5701 "Good Octets Received");
5702 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5703 CTLFLAG_RD, &stats->gotc,
5704 "Good Octets Transmitted");
5705 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5706 CTLFLAG_RD, &stats->mprc,
5707 "Multicast Packets Received");
5711 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5712 CTLFLAG_RD, &stats->ecol,
5713 "Excessive collisions");
5714 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
5715 CTLFLAG_RD, &stats->scc,
5716 "Single collisions");
5717 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5718 CTLFLAG_RD, &stats->mcc,
5719 "Multiple collisions");
5720 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
5721 CTLFLAG_RD, &stats->latecol,
5723 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
5724 CTLFLAG_RD, &stats->colc,
5726 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5727 CTLFLAG_RD, &stats->symerrs,
5729 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5730 CTLFLAG_RD, &stats->sec,
5732 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
5733 CTLFLAG_RD, &stats->dc,
5735 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5736 CTLFLAG_RD, &stats->mpc,
5738 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5739 CTLFLAG_RD, &stats->rnbc,
5740 "Receive No Buffers");
5741 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5742 CTLFLAG_RD, &stats->ruc,
5743 "Receive Undersize");
5744 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5745 CTLFLAG_RD, &stats->rfc,
5746 "Fragmented Packets Received ");
5747 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5748 CTLFLAG_RD, &stats->roc,
5749 "Oversized Packets Received");
5750 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5751 CTLFLAG_RD, &stats->rjc,
5753 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5754 CTLFLAG_RD, &stats->rxerrc,
5756 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5757 CTLFLAG_RD, &stats->crcerrs,
5759 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5760 CTLFLAG_RD, &stats->algnerrc,
5761 "Alignment Errors");
5762 /* On 82575 these are collision counts */
5763 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5764 CTLFLAG_RD, &stats->cexterr,
5765 "Collision/Carrier extension errors");
5766 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5767 CTLFLAG_RD, &stats->xonrxc,
5769 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5770 CTLFLAG_RD, &stats->xontxc,
5772 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5773 CTLFLAG_RD, &stats->xoffrxc,
5775 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5776 CTLFLAG_RD, &stats->xofftxc,
5777 "XOFF Transmitted");
5778 /* Packet Reception Stats */
5779 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5780 CTLFLAG_RD, &stats->tpr,
5781 "Total Packets Received ");
5782 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5783 CTLFLAG_RD, &stats->gprc,
5784 "Good Packets Received");
5785 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5786 CTLFLAG_RD, &stats->bprc,
5787 "Broadcast Packets Received");
5788 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5789 CTLFLAG_RD, &stats->mprc,
5790 "Multicast Packets Received");
5791 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5792 CTLFLAG_RD, &stats->prc64,
5793 "64 byte frames received ");
5794 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5795 CTLFLAG_RD, &stats->prc127,
5796 "65-127 byte frames received");
5797 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5798 CTLFLAG_RD, &stats->prc255,
5799 "128-255 byte frames received");
5800 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5801 CTLFLAG_RD, &stats->prc511,
5802 "256-511 byte frames received");
5803 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5804 CTLFLAG_RD, &stats->prc1023,
5805 "512-1023 byte frames received");
5806 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5807 CTLFLAG_RD, &stats->prc1522,
5808 "1023-1522 byte frames received");
5809 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5810 CTLFLAG_RD, &stats->gorc,
5811 "Good Octets Received");
5813 /* Packet Transmission Stats */
5814 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5815 CTLFLAG_RD, &stats->gotc,
5816 "Good Octets Transmitted");
5817 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5818 CTLFLAG_RD, &stats->tpt,
5819 "Total Packets Transmitted");
5820 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5821 CTLFLAG_RD, &stats->gptc,
5822 "Good Packets Transmitted");
5823 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5824 CTLFLAG_RD, &stats->bptc,
5825 "Broadcast Packets Transmitted");
5826 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5827 CTLFLAG_RD, &stats->mptc,
5828 "Multicast Packets Transmitted");
5829 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5830 CTLFLAG_RD, &stats->ptc64,
5831 "64 byte frames transmitted ");
5832 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5833 CTLFLAG_RD, &stats->ptc127,
5834 "65-127 byte frames transmitted");
5835 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5836 CTLFLAG_RD, &stats->ptc255,
5837 "128-255 byte frames transmitted");
5838 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5839 CTLFLAG_RD, &stats->ptc511,
5840 "256-511 byte frames transmitted");
5841 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5842 CTLFLAG_RD, &stats->ptc1023,
5843 "512-1023 byte frames transmitted");
5844 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5845 CTLFLAG_RD, &stats->ptc1522,
5846 "1024-1522 byte frames transmitted");
5847 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5848 CTLFLAG_RD, &stats->tsctc,
5849 "TSO Contexts Transmitted");
5850 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5851 CTLFLAG_RD, &stats->tsctfc,
5852 "TSO Contexts Failed");
5855 /* Interrupt Stats */
5857 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5858 CTLFLAG_RD, NULL, "Interrupt Statistics");
5859 int_list = SYSCTL_CHILDREN(int_node);
5861 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
5862 CTLFLAG_RD, &stats->iac,
5863 "Interrupt Assertion Count");
5865 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5866 CTLFLAG_RD, &stats->icrxptc,
5867 "Interrupt Cause Rx Pkt Timer Expire Count");
5869 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5870 CTLFLAG_RD, &stats->icrxatc,
5871 "Interrupt Cause Rx Abs Timer Expire Count");
5873 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5874 CTLFLAG_RD, &stats->ictxptc,
5875 "Interrupt Cause Tx Pkt Timer Expire Count");
5877 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5878 CTLFLAG_RD, &stats->ictxatc,
5879 "Interrupt Cause Tx Abs Timer Expire Count");
5881 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5882 CTLFLAG_RD, &stats->ictxqec,
5883 "Interrupt Cause Tx Queue Empty Count");
5885 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5886 CTLFLAG_RD, &stats->ictxqmtc,
5887 "Interrupt Cause Tx Queue Min Thresh Count");
5889 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5890 CTLFLAG_RD, &stats->icrxdmtc,
5891 "Interrupt Cause Rx Desc Min Thresh Count");
5893 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5894 CTLFLAG_RD, &stats->icrxoc,
5895 "Interrupt Cause Receiver Overrun Count");
5897 /* Host to Card Stats */
5899 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
5901 "Host to Card Statistics");
5903 host_list = SYSCTL_CHILDREN(host_node);
5905 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
5906 CTLFLAG_RD, &stats->cbtmpc,
5907 "Circuit Breaker Tx Packet Count");
5909 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
5910 CTLFLAG_RD, &stats->htdpmc,
5911 "Host Transmit Discarded Packets");
5913 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
5914 CTLFLAG_RD, &stats->rpthc,
5915 "Rx Packets To Host");
5917 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
5918 CTLFLAG_RD, &stats->cbrmpc,
5919 "Circuit Breaker Rx Packet Count");
5921 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
5922 CTLFLAG_RD, &stats->cbrdpc,
5923 "Circuit Breaker Rx Dropped Count");
5925 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
5926 CTLFLAG_RD, &stats->hgptc,
5927 "Host Good Packets Tx Count");
5929 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
5930 CTLFLAG_RD, &stats->htcbdpc,
5931 "Host Tx Circuit Breaker Dropped Count");
5933 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
5934 CTLFLAG_RD, &stats->hgorc,
5935 "Host Good Octets Received Count");
5937 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
5938 CTLFLAG_RD, &stats->hgotc,
5939 "Host Good Octets Transmit Count");
5941 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
5942 CTLFLAG_RD, &stats->lenerrs,
5945 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
5946 CTLFLAG_RD, &stats->scvpc,
5947 "SerDes/SGMII Code Violation Pkt Count");
5949 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
5950 CTLFLAG_RD, &stats->hrmpc,
5951 "Header Redirection Missed Packet Count");
5955 /**********************************************************************
5957 * This routine provides a way to dump out the adapter eeprom,
5958 * often a useful debug/service tool. This only dumps the first
5959 * 32 words, stuff that matters is in that extent.
5961 **********************************************************************/
5963 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5965 struct adapter *adapter;
5970 error = sysctl_handle_int(oidp, &result, 0, req);
5972 if (error || !req->newptr)
5976 * This value will cause a hex dump of the
5977 * first 32 16-bit words of the EEPROM to
5981 adapter = (struct adapter *)arg1;
5982 igb_print_nvm_info(adapter);
5989 igb_print_nvm_info(struct adapter *adapter)
5994 /* Its a bit crude, but it gets the job done */
5995 printf("\nInterface EEPROM Dump:\n");
5996 printf("Offset\n0x0000 ");
5997 for (i = 0, j = 0; i < 32; i++, j++) {
5998 if (j == 8) { /* Make the offset block */
6000 printf("\n0x00%x0 ",row);
6002 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
6003 printf("%04x ", eeprom_data);
6009 igb_set_sysctl_value(struct adapter *adapter, const char *name,
6010 const char *description, int *limit, int value)
6013 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
6014 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
6015 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
6019 ** Set flow control using sysctl:
6020 ** Flow control values:
6027 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
6030 static int input = 3; /* default is full */
6031 struct adapter *adapter = (struct adapter *) arg1;
6033 error = sysctl_handle_int(oidp, &input, 0, req);
6035 if ((error) || (req->newptr == NULL))
6039 case e1000_fc_rx_pause:
6040 case e1000_fc_tx_pause:
6043 adapter->hw.fc.requested_mode = input;
6044 adapter->fc = input;
6051 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
6052 e1000_force_mac_fc(&adapter->hw);
6057 ** Manage DMA Coalesce:
6060 ** Legal timer values are:
6061 ** 250,500,1000-10000 in thousands
6064 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
6066 struct adapter *adapter = (struct adapter *) arg1;
6069 error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
6071 if ((error) || (req->newptr == NULL))
6074 switch (adapter->dmac) {
6078 case 1: /* Just enable and use default */
6079 adapter->dmac = 1000;
6093 /* Legal values - allow */
6096 /* Do nothing, illegal value */
6100 /* Reinit the interface */
6106 ** Manage Energy Efficient Ethernet:
6108 ** 0/1 - enabled/disabled
6111 igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
6113 struct adapter *adapter = (struct adapter *) arg1;
6116 value = adapter->hw.dev_spec._82575.eee_disable;
6117 error = sysctl_handle_int(oidp, &value, 0, req);
6118 if (error || req->newptr == NULL)
6120 IGB_CORE_LOCK(adapter);
6121 adapter->hw.dev_spec._82575.eee_disable = (value != 0);
6122 igb_init_locked(adapter);
6123 IGB_CORE_UNLOCK(adapter);