1 /******************************************************************************
3 Copyright (c) 2001-2013, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
39 #ifdef HAVE_KERNEL_OPTION_HEADERS
40 #include "opt_device_polling.h"
44 #include <sys/param.h>
45 #include <sys/systm.h>
47 #include <sys/buf_ring.h>
50 #include <sys/endian.h>
51 #include <sys/kernel.h>
52 #include <sys/kthread.h>
53 #include <sys/malloc.h>
55 #include <sys/module.h>
57 #include <sys/socket.h>
58 #include <sys/sockio.h>
59 #include <sys/sysctl.h>
60 #include <sys/taskqueue.h>
61 #include <sys/eventhandler.h>
64 #include <machine/smp.h>
65 #include <machine/bus.h>
66 #include <machine/resource.h>
69 #include <net/ethernet.h>
71 #include <net/if_arp.h>
72 #include <net/if_dl.h>
73 #include <net/if_media.h>
75 #include <net/if_types.h>
76 #include <net/if_vlan_var.h>
78 #include <netinet/in_systm.h>
79 #include <netinet/in.h>
80 #include <netinet/if_ether.h>
81 #include <netinet/ip.h>
82 #include <netinet/ip6.h>
83 #include <netinet/tcp.h>
84 #include <netinet/tcp_lro.h>
85 #include <netinet/udp.h>
87 #include <machine/in_cksum.h>
88 #include <dev/led/led.h>
89 #include <dev/pci/pcivar.h>
90 #include <dev/pci/pcireg.h>
92 #include "e1000_api.h"
93 #include "e1000_82575.h"
96 /*********************************************************************
97 * Set this to one to display debug statistics
98 *********************************************************************/
99 int igb_display_debug_stats = 0;
101 /*********************************************************************
103 *********************************************************************/
104 char igb_driver_version[] = "version - 2.4.0";
107 /*********************************************************************
108 * PCI Device ID Table
110 * Used by probe to select devices to load on
111 * Last field stores an index into e1000_strings
112 * Last entry must be all 0s
114 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
115 *********************************************************************/
117 static igb_vendor_info_t igb_vendor_info_array[] =
119 { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
120 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
121 PCI_ANY_ID, PCI_ANY_ID, 0},
122 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
123 PCI_ANY_ID, PCI_ANY_ID, 0},
124 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0},
126 { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
128 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
129 { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
130 PCI_ANY_ID, PCI_ANY_ID, 0},
131 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
132 PCI_ANY_ID, PCI_ANY_ID, 0},
133 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2,
134 PCI_ANY_ID, PCI_ANY_ID, 0},
135 { 0x8086, E1000_DEV_ID_82576_VF, PCI_ANY_ID, PCI_ANY_ID, 0},
136 { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
137 { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
138 { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
139 { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
140 { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL,
141 PCI_ANY_ID, PCI_ANY_ID, 0},
142 { 0x8086, E1000_DEV_ID_82580_QUAD_FIBER,
143 PCI_ANY_ID, PCI_ANY_ID, 0},
144 { 0x8086, E1000_DEV_ID_DH89XXCC_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
145 { 0x8086, E1000_DEV_ID_DH89XXCC_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_DH89XXCC_SFP, PCI_ANY_ID, PCI_ANY_ID, 0},
147 { 0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE,
148 PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_I350_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_I350_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
151 { 0x8086, E1000_DEV_ID_I350_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
152 { 0x8086, E1000_DEV_ID_I350_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
153 { 0x8086, E1000_DEV_ID_I350_VF, PCI_ANY_ID, PCI_ANY_ID, 0},
154 { 0x8086, E1000_DEV_ID_I210_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
155 { 0x8086, E1000_DEV_ID_I210_COPPER_IT, PCI_ANY_ID, PCI_ANY_ID, 0},
156 { 0x8086, E1000_DEV_ID_I210_COPPER_OEM1,
157 PCI_ANY_ID, PCI_ANY_ID, 0},
158 { 0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS,
159 PCI_ANY_ID, PCI_ANY_ID, 0},
160 { 0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS,
161 PCI_ANY_ID, PCI_ANY_ID, 0},
162 { 0x8086, E1000_DEV_ID_I210_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
163 { 0x8086, E1000_DEV_ID_I210_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
164 { 0x8086, E1000_DEV_ID_I210_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
165 { 0x8086, E1000_DEV_ID_I211_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
166 { 0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS,
167 PCI_ANY_ID, PCI_ANY_ID, 0},
168 { 0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS,
169 PCI_ANY_ID, PCI_ANY_ID, 0},
170 { 0x8086, E1000_DEV_ID_I354_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0},
171 /* required last entry */
175 /*********************************************************************
176 * Table of branding strings for all supported NICs.
177 *********************************************************************/
179 static char *igb_strings[] = {
180 "Intel(R) PRO/1000 Network Connection"
183 /*********************************************************************
184 * Function prototypes
185 *********************************************************************/
186 static int igb_probe(device_t);
187 static int igb_attach(device_t);
188 static int igb_detach(device_t);
189 static int igb_shutdown(device_t);
190 static int igb_suspend(device_t);
191 static int igb_resume(device_t);
192 #ifndef IGB_LEGACY_TX
193 static int igb_mq_start(struct ifnet *, struct mbuf *);
194 static int igb_mq_start_locked(struct ifnet *, struct tx_ring *);
195 static void igb_qflush(struct ifnet *);
196 static void igb_deferred_mq_start(void *, int);
198 static void igb_start(struct ifnet *);
199 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
201 static int igb_ioctl(struct ifnet *, u_long, caddr_t);
202 static void igb_init(void *);
203 static void igb_init_locked(struct adapter *);
204 static void igb_stop(void *);
205 static void igb_media_status(struct ifnet *, struct ifmediareq *);
206 static int igb_media_change(struct ifnet *);
207 static void igb_identify_hardware(struct adapter *);
208 static int igb_allocate_pci_resources(struct adapter *);
209 static int igb_allocate_msix(struct adapter *);
210 static int igb_allocate_legacy(struct adapter *);
211 static int igb_setup_msix(struct adapter *);
212 static void igb_free_pci_resources(struct adapter *);
213 static void igb_local_timer(void *);
214 static void igb_reset(struct adapter *);
215 static int igb_setup_interface(device_t, struct adapter *);
216 static int igb_allocate_queues(struct adapter *);
217 static void igb_configure_queues(struct adapter *);
219 static int igb_allocate_transmit_buffers(struct tx_ring *);
220 static void igb_setup_transmit_structures(struct adapter *);
221 static void igb_setup_transmit_ring(struct tx_ring *);
222 static void igb_initialize_transmit_units(struct adapter *);
223 static void igb_free_transmit_structures(struct adapter *);
224 static void igb_free_transmit_buffers(struct tx_ring *);
226 static int igb_allocate_receive_buffers(struct rx_ring *);
227 static int igb_setup_receive_structures(struct adapter *);
228 static int igb_setup_receive_ring(struct rx_ring *);
229 static void igb_initialize_receive_units(struct adapter *);
230 static void igb_free_receive_structures(struct adapter *);
231 static void igb_free_receive_buffers(struct rx_ring *);
232 static void igb_free_receive_ring(struct rx_ring *);
234 static void igb_enable_intr(struct adapter *);
235 static void igb_disable_intr(struct adapter *);
236 static void igb_update_stats_counters(struct adapter *);
237 static bool igb_txeof(struct tx_ring *);
239 static __inline void igb_rx_discard(struct rx_ring *, int);
240 static __inline void igb_rx_input(struct rx_ring *,
241 struct ifnet *, struct mbuf *, u32);
243 static bool igb_rxeof(struct igb_queue *, int, int *);
244 static void igb_rx_checksum(u32, struct mbuf *, u32);
245 static int igb_tx_ctx_setup(struct tx_ring *,
246 struct mbuf *, u32 *, u32 *);
247 static int igb_tso_setup(struct tx_ring *,
248 struct mbuf *, u32 *, u32 *);
249 static void igb_set_promisc(struct adapter *);
250 static void igb_disable_promisc(struct adapter *);
251 static void igb_set_multi(struct adapter *);
252 static void igb_update_link_status(struct adapter *);
253 static void igb_refresh_mbufs(struct rx_ring *, int);
255 static void igb_register_vlan(void *, struct ifnet *, u16);
256 static void igb_unregister_vlan(void *, struct ifnet *, u16);
257 static void igb_setup_vlan_hw_support(struct adapter *);
259 static int igb_xmit(struct tx_ring *, struct mbuf **);
260 static int igb_dma_malloc(struct adapter *, bus_size_t,
261 struct igb_dma_alloc *, int);
262 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
263 static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
264 static void igb_print_nvm_info(struct adapter *);
265 static int igb_is_valid_ether_addr(u8 *);
266 static void igb_add_hw_stats(struct adapter *);
268 static void igb_vf_init_stats(struct adapter *);
269 static void igb_update_vf_stats_counters(struct adapter *);
271 /* Management and WOL Support */
272 static void igb_init_manageability(struct adapter *);
273 static void igb_release_manageability(struct adapter *);
274 static void igb_get_hw_control(struct adapter *);
275 static void igb_release_hw_control(struct adapter *);
276 static void igb_enable_wakeup(device_t);
277 static void igb_led_func(void *, int);
279 static int igb_irq_fast(void *);
280 static void igb_msix_que(void *);
281 static void igb_msix_link(void *);
282 static void igb_handle_que(void *context, int pending);
283 static void igb_handle_link(void *context, int pending);
284 static void igb_handle_link_locked(struct adapter *);
286 static void igb_set_sysctl_value(struct adapter *, const char *,
287 const char *, int *, int);
288 static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
289 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
290 static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
292 #ifdef DEVICE_POLLING
293 static poll_handler_t igb_poll;
296 /*********************************************************************
297 * FreeBSD Device Interface Entry Points
298 *********************************************************************/
300 static device_method_t igb_methods[] = {
301 /* Device interface */
302 DEVMETHOD(device_probe, igb_probe),
303 DEVMETHOD(device_attach, igb_attach),
304 DEVMETHOD(device_detach, igb_detach),
305 DEVMETHOD(device_shutdown, igb_shutdown),
306 DEVMETHOD(device_suspend, igb_suspend),
307 DEVMETHOD(device_resume, igb_resume),
311 static driver_t igb_driver = {
312 "igb", igb_methods, sizeof(struct adapter),
315 static devclass_t igb_devclass;
316 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
317 MODULE_DEPEND(igb, pci, 1, 1, 1);
318 MODULE_DEPEND(igb, ether, 1, 1, 1);
320 /*********************************************************************
321 * Tunable default values.
322 *********************************************************************/
324 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
326 /* Descriptor defaults */
327 static int igb_rxd = IGB_DEFAULT_RXD;
328 static int igb_txd = IGB_DEFAULT_TXD;
329 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
330 TUNABLE_INT("hw.igb.txd", &igb_txd);
331 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
332 "Number of receive descriptors per queue");
333 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
334 "Number of transmit descriptors per queue");
337 ** AIM: Adaptive Interrupt Moderation
338 ** which means that the interrupt rate
339 ** is varied over time based on the
340 ** traffic for that interrupt vector
342 static int igb_enable_aim = TRUE;
343 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
344 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0,
345 "Enable adaptive interrupt moderation");
348 * MSIX should be the default for best performance,
349 * but this allows it to be forced off for testing.
351 static int igb_enable_msix = 1;
352 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
353 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
354 "Enable MSI-X interrupts");
357 ** Tuneable Interrupt rate
359 static int igb_max_interrupt_rate = 8000;
360 TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate);
361 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
362 &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
364 #ifndef IGB_LEGACY_TX
366 ** Tuneable number of buffers in the buf-ring (drbr_xxx)
368 static int igb_buf_ring_size = IGB_BR_SIZE;
369 TUNABLE_INT("hw.igb.buf_ring_size", &igb_buf_ring_size);
370 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
371 &igb_buf_ring_size, 0, "Size of the bufring");
375 ** Header split causes the packet header to
376 ** be dma'd to a seperate mbuf from the payload.
377 ** this can have memory alignment benefits. But
378 ** another plus is that small packets often fit
379 ** into the header and thus use no cluster. Its
380 ** a very workload dependent type feature.
382 static int igb_header_split = FALSE;
383 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
384 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
385 "Enable receive mbuf header split");
388 ** This will autoconfigure based on the
389 ** number of CPUs and max supported
390 ** MSIX messages if left at 0.
392 static int igb_num_queues = 0;
393 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
394 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
395 "Number of queues to configure, 0 indicates autoconfigure");
398 ** Global variable to store last used CPU when binding queues
399 ** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
400 ** queue is bound to a cpu.
402 static int igb_last_bind_cpu = -1;
404 /* How many packets rxeof tries to clean at a time */
405 static int igb_rx_process_limit = 100;
406 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
407 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
408 &igb_rx_process_limit, 0,
409 "Maximum number of received packets to process at a time, -1 means unlimited");
411 #ifdef DEV_NETMAP /* see ixgbe.c for details */
412 #include <dev/netmap/if_igb_netmap.h>
413 #endif /* DEV_NETMAP */
414 /*********************************************************************
415 * Device identification routine
417 * igb_probe determines if the driver should be loaded on
418 * adapter based on PCI vendor/device id of the adapter.
420 * return BUS_PROBE_DEFAULT on success, positive on failure
421 *********************************************************************/
424 igb_probe(device_t dev)
426 char adapter_name[60];
427 uint16_t pci_vendor_id = 0;
428 uint16_t pci_device_id = 0;
429 uint16_t pci_subvendor_id = 0;
430 uint16_t pci_subdevice_id = 0;
431 igb_vendor_info_t *ent;
433 INIT_DEBUGOUT("igb_probe: begin");
435 pci_vendor_id = pci_get_vendor(dev);
436 if (pci_vendor_id != IGB_VENDOR_ID)
439 pci_device_id = pci_get_device(dev);
440 pci_subvendor_id = pci_get_subvendor(dev);
441 pci_subdevice_id = pci_get_subdevice(dev);
443 ent = igb_vendor_info_array;
444 while (ent->vendor_id != 0) {
445 if ((pci_vendor_id == ent->vendor_id) &&
446 (pci_device_id == ent->device_id) &&
448 ((pci_subvendor_id == ent->subvendor_id) ||
449 (ent->subvendor_id == PCI_ANY_ID)) &&
451 ((pci_subdevice_id == ent->subdevice_id) ||
452 (ent->subdevice_id == PCI_ANY_ID))) {
453 sprintf(adapter_name, "%s %s",
454 igb_strings[ent->index],
456 device_set_desc_copy(dev, adapter_name);
457 return (BUS_PROBE_DEFAULT);
465 /*********************************************************************
466 * Device initialization routine
468 * The attach entry point is called when the driver is being loaded.
469 * This routine identifies the type of hardware, allocates all resources
470 * and initializes the hardware.
472 * return 0 on success, positive on failure
473 *********************************************************************/
476 igb_attach(device_t dev)
478 struct adapter *adapter;
482 INIT_DEBUGOUT("igb_attach: begin");
484 if (resource_disabled("igb", device_get_unit(dev))) {
485 device_printf(dev, "Disabled by device hint\n");
489 adapter = device_get_softc(dev);
490 adapter->dev = adapter->osdep.dev = dev;
491 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
494 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
495 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
496 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
497 igb_sysctl_nvm_info, "I", "NVM Information");
499 igb_set_sysctl_value(adapter, "enable_aim",
500 "Interrupt Moderation", &adapter->enable_aim,
503 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
504 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
505 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
506 adapter, 0, igb_set_flowcntl, "I", "Flow Control");
508 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
510 /* Determine hardware and mac info */
511 igb_identify_hardware(adapter);
513 /* Setup PCI resources */
514 if (igb_allocate_pci_resources(adapter)) {
515 device_printf(dev, "Allocation of PCI resources failed\n");
520 /* Do Shared Code initialization */
521 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
522 device_printf(dev, "Setup of Shared code failed\n");
527 e1000_get_bus_info(&adapter->hw);
529 /* Sysctl for limiting the amount of work done in the taskqueue */
530 igb_set_sysctl_value(adapter, "rx_processing_limit",
531 "max number of rx packets to process",
532 &adapter->rx_process_limit, igb_rx_process_limit);
535 * Validate number of transmit and receive descriptors. It
536 * must not exceed hardware maximum, and must be multiple
537 * of E1000_DBA_ALIGN.
539 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
540 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
541 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
542 IGB_DEFAULT_TXD, igb_txd);
543 adapter->num_tx_desc = IGB_DEFAULT_TXD;
545 adapter->num_tx_desc = igb_txd;
546 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
547 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
548 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
549 IGB_DEFAULT_RXD, igb_rxd);
550 adapter->num_rx_desc = IGB_DEFAULT_RXD;
552 adapter->num_rx_desc = igb_rxd;
554 adapter->hw.mac.autoneg = DO_AUTO_NEG;
555 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
556 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
559 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
560 adapter->hw.phy.mdix = AUTO_ALL_MODES;
561 adapter->hw.phy.disable_polarity_correction = FALSE;
562 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
566 * Set the frame limits assuming
567 * standard ethernet sized frames.
569 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
572 ** Allocate and Setup Queues
574 if (igb_allocate_queues(adapter)) {
579 /* Allocate the appropriate stats memory */
580 if (adapter->vf_ifp) {
582 (struct e1000_vf_stats *)malloc(sizeof \
583 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
584 igb_vf_init_stats(adapter);
587 (struct e1000_hw_stats *)malloc(sizeof \
588 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
589 if (adapter->stats == NULL) {
590 device_printf(dev, "Can not allocate stats memory\n");
595 /* Allocate multicast array memory. */
596 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
597 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
598 if (adapter->mta == NULL) {
599 device_printf(dev, "Can not allocate multicast setup array\n");
604 /* Some adapter-specific advanced features */
605 if (adapter->hw.mac.type >= e1000_i350) {
606 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
607 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
608 OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
609 adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
610 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
611 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
612 OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
613 adapter, 0, igb_sysctl_eee, "I",
614 "Disable Energy Efficient Ethernet");
615 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
616 if (adapter->hw.mac.type == e1000_i354)
617 e1000_set_eee_i354(&adapter->hw);
619 e1000_set_eee_i350(&adapter->hw);
624 ** Start from a known state, this is
625 ** important in reading the nvm and
628 e1000_reset_hw(&adapter->hw);
630 /* Make sure we have a good EEPROM before we read from it */
631 if (((adapter->hw.mac.type != e1000_i210) &&
632 (adapter->hw.mac.type != e1000_i211)) &&
633 (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
635 ** Some PCI-E parts fail the first check due to
636 ** the link being in sleep state, call it again,
637 ** if it fails a second time its a real issue.
639 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
641 "The EEPROM Checksum Is Not Valid\n");
648 ** Copy the permanent MAC address out of the EEPROM
650 if (e1000_read_mac_addr(&adapter->hw) < 0) {
651 device_printf(dev, "EEPROM read error while reading MAC"
656 /* Check its sanity */
657 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
658 device_printf(dev, "Invalid MAC address\n");
663 /* Setup OS specific network interface */
664 if (igb_setup_interface(dev, adapter) != 0)
667 /* Now get a good starting state */
670 /* Initialize statistics */
671 igb_update_stats_counters(adapter);
673 adapter->hw.mac.get_link_status = 1;
674 igb_update_link_status(adapter);
676 /* Indicate SOL/IDER usage */
677 if (e1000_check_reset_block(&adapter->hw))
679 "PHY reset is blocked due to SOL/IDER session.\n");
681 /* Determine if we have to control management hardware */
682 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
687 /* APME bit in EEPROM is mapped to WUC.APME */
688 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
690 adapter->wol = E1000_WUFC_MAG;
692 /* Register for VLAN events */
693 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
694 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
695 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
696 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
698 igb_add_hw_stats(adapter);
700 /* Tell the stack that the interface is not active */
701 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
702 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
704 adapter->led_dev = led_create(igb_led_func, adapter,
705 device_get_nameunit(dev));
708 ** Configure Interrupts
710 if ((adapter->msix > 1) && (igb_enable_msix))
711 error = igb_allocate_msix(adapter);
712 else /* MSI or Legacy */
713 error = igb_allocate_legacy(adapter);
718 igb_netmap_attach(adapter);
719 #endif /* DEV_NETMAP */
720 INIT_DEBUGOUT("igb_attach: end");
726 igb_free_transmit_structures(adapter);
727 igb_free_receive_structures(adapter);
728 igb_release_hw_control(adapter);
730 igb_free_pci_resources(adapter);
731 if (adapter->ifp != NULL)
732 if_free(adapter->ifp);
733 free(adapter->mta, M_DEVBUF);
734 IGB_CORE_LOCK_DESTROY(adapter);
739 /*********************************************************************
740 * Device removal routine
742 * The detach entry point is called when the driver is being removed.
743 * This routine stops the adapter and deallocates all the resources
744 * that were allocated for driver operation.
746 * return 0 on success, positive on failure
747 *********************************************************************/
750 igb_detach(device_t dev)
752 struct adapter *adapter = device_get_softc(dev);
753 struct ifnet *ifp = adapter->ifp;
755 INIT_DEBUGOUT("igb_detach: begin");
757 /* Make sure VLANS are not using driver */
758 if (adapter->ifp->if_vlantrunk != NULL) {
759 device_printf(dev,"Vlan in use, detach first\n");
763 ether_ifdetach(adapter->ifp);
765 if (adapter->led_dev != NULL)
766 led_destroy(adapter->led_dev);
768 #ifdef DEVICE_POLLING
769 if (ifp->if_capenable & IFCAP_POLLING)
770 ether_poll_deregister(ifp);
773 IGB_CORE_LOCK(adapter);
774 adapter->in_detach = 1;
776 IGB_CORE_UNLOCK(adapter);
778 e1000_phy_hw_reset(&adapter->hw);
780 /* Give control back to firmware */
781 igb_release_manageability(adapter);
782 igb_release_hw_control(adapter);
785 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
786 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
787 igb_enable_wakeup(dev);
790 /* Unregister VLAN events */
791 if (adapter->vlan_attach != NULL)
792 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
793 if (adapter->vlan_detach != NULL)
794 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
796 callout_drain(&adapter->timer);
799 netmap_detach(adapter->ifp);
800 #endif /* DEV_NETMAP */
801 igb_free_pci_resources(adapter);
802 bus_generic_detach(dev);
805 igb_free_transmit_structures(adapter);
806 igb_free_receive_structures(adapter);
807 if (adapter->mta != NULL)
808 free(adapter->mta, M_DEVBUF);
810 IGB_CORE_LOCK_DESTROY(adapter);
815 /*********************************************************************
817 * Shutdown entry point
819 **********************************************************************/
822 igb_shutdown(device_t dev)
824 return igb_suspend(dev);
828 * Suspend/resume device methods.
831 igb_suspend(device_t dev)
833 struct adapter *adapter = device_get_softc(dev);
835 IGB_CORE_LOCK(adapter);
839 igb_release_manageability(adapter);
840 igb_release_hw_control(adapter);
843 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
844 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
845 igb_enable_wakeup(dev);
848 IGB_CORE_UNLOCK(adapter);
850 return bus_generic_suspend(dev);
854 igb_resume(device_t dev)
856 struct adapter *adapter = device_get_softc(dev);
857 struct tx_ring *txr = adapter->tx_rings;
858 struct ifnet *ifp = adapter->ifp;
860 IGB_CORE_LOCK(adapter);
861 igb_init_locked(adapter);
862 igb_init_manageability(adapter);
864 if ((ifp->if_flags & IFF_UP) &&
865 (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
866 for (int i = 0; i < adapter->num_queues; i++, txr++) {
868 #ifndef IGB_LEGACY_TX
869 /* Process the stack queue only if not depleted */
870 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
871 !drbr_empty(ifp, txr->br))
872 igb_mq_start_locked(ifp, txr);
874 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
875 igb_start_locked(txr, ifp);
880 IGB_CORE_UNLOCK(adapter);
882 return bus_generic_resume(dev);
888 /*********************************************************************
889 * Transmit entry point
891 * igb_start is called by the stack to initiate a transmit.
892 * The driver will remain in this routine as long as there are
893 * packets to transmit and transmit resources are available.
894 * In case resources are not available stack is notified and
895 * the packet is requeued.
896 **********************************************************************/
899 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
901 struct adapter *adapter = ifp->if_softc;
904 IGB_TX_LOCK_ASSERT(txr);
906 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
909 if (!adapter->link_active)
912 /* Call cleanup if number of TX descriptors low */
913 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
916 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
917 if (txr->tx_avail <= IGB_MAX_SCATTER) {
918 txr->queue_status |= IGB_QUEUE_DEPLETED;
921 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
925 * Encapsulation can modify our pointer, and or make it
926 * NULL on failure. In that event, we can't requeue.
928 if (igb_xmit(txr, &m_head)) {
930 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
931 if (txr->tx_avail <= IGB_MAX_SCATTER)
932 txr->queue_status |= IGB_QUEUE_DEPLETED;
936 /* Send a copy of the frame to the BPF listener */
937 ETHER_BPF_MTAP(ifp, m_head);
939 /* Set watchdog on */
940 txr->watchdog_time = ticks;
941 txr->queue_status |= IGB_QUEUE_WORKING;
946 * Legacy TX driver routine, called from the
947 * stack, always uses tx[0], and spins for it.
948 * Should not be used with multiqueue tx
951 igb_start(struct ifnet *ifp)
953 struct adapter *adapter = ifp->if_softc;
954 struct tx_ring *txr = adapter->tx_rings;
956 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
958 igb_start_locked(txr, ifp);
964 #else /* ~IGB_LEGACY_TX */
967 ** Multiqueue Transmit Entry:
968 ** quick turnaround to the stack
972 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
974 struct adapter *adapter = ifp->if_softc;
975 struct igb_queue *que;
979 /* Which queue to use */
980 if ((m->m_flags & M_FLOWID) != 0)
981 i = m->m_pkthdr.flowid % adapter->num_queues;
983 i = curcpu % adapter->num_queues;
984 txr = &adapter->tx_rings[i];
985 que = &adapter->queues[i];
987 err = drbr_enqueue(ifp, txr->br, m);
990 if (IGB_TX_TRYLOCK(txr)) {
991 igb_mq_start_locked(ifp, txr);
994 taskqueue_enqueue(que->tq, &txr->txq_task);
1000 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
1002 struct adapter *adapter = txr->adapter;
1004 int err = 0, enq = 0;
1006 IGB_TX_LOCK_ASSERT(txr);
1008 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
1009 adapter->link_active == 0)
1013 /* Process the queue */
1014 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
1015 if ((err = igb_xmit(txr, &next)) != 0) {
1017 /* It was freed, move forward */
1018 drbr_advance(ifp, txr->br);
1021 * Still have one left, it may not be
1022 * the same since the transmit function
1023 * may have changed it.
1025 drbr_putback(ifp, txr->br, next);
1029 drbr_advance(ifp, txr->br);
1031 ifp->if_obytes += next->m_pkthdr.len;
1032 if (next->m_flags & M_MCAST)
1034 ETHER_BPF_MTAP(ifp, next);
1035 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1039 /* Set the watchdog */
1040 txr->queue_status |= IGB_QUEUE_WORKING;
1041 txr->watchdog_time = ticks;
1043 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
1045 if (txr->tx_avail <= IGB_MAX_SCATTER)
1046 txr->queue_status |= IGB_QUEUE_DEPLETED;
1051 * Called from a taskqueue to drain queued transmit packets.
1054 igb_deferred_mq_start(void *arg, int pending)
1056 struct tx_ring *txr = arg;
1057 struct adapter *adapter = txr->adapter;
1058 struct ifnet *ifp = adapter->ifp;
1061 if (!drbr_empty(ifp, txr->br))
1062 igb_mq_start_locked(ifp, txr);
1067 ** Flush all ring buffers
1070 igb_qflush(struct ifnet *ifp)
1072 struct adapter *adapter = ifp->if_softc;
1073 struct tx_ring *txr = adapter->tx_rings;
1076 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1078 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1084 #endif /* ~IGB_LEGACY_TX */
1086 /*********************************************************************
1089 * igb_ioctl is called when the user wants to configure the
1092 * return 0 on success, positive on failure
1093 **********************************************************************/
1096 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1098 struct adapter *adapter = ifp->if_softc;
1099 struct ifreq *ifr = (struct ifreq *)data;
1100 #if defined(INET) || defined(INET6)
1101 struct ifaddr *ifa = (struct ifaddr *)data;
1103 bool avoid_reset = FALSE;
1106 if (adapter->in_detach)
1112 if (ifa->ifa_addr->sa_family == AF_INET)
1116 if (ifa->ifa_addr->sa_family == AF_INET6)
1120 ** Calling init results in link renegotiation,
1121 ** so we avoid doing it when possible.
1124 ifp->if_flags |= IFF_UP;
1125 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1128 if (!(ifp->if_flags & IFF_NOARP))
1129 arp_ifinit(ifp, ifa);
1132 error = ether_ioctl(ifp, command, data);
1138 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1140 IGB_CORE_LOCK(adapter);
1141 max_frame_size = 9234;
1142 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1144 IGB_CORE_UNLOCK(adapter);
1149 ifp->if_mtu = ifr->ifr_mtu;
1150 adapter->max_frame_size =
1151 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1152 igb_init_locked(adapter);
1153 IGB_CORE_UNLOCK(adapter);
1157 IOCTL_DEBUGOUT("ioctl rcv'd:\
1158 SIOCSIFFLAGS (Set Interface Flags)");
1159 IGB_CORE_LOCK(adapter);
1160 if (ifp->if_flags & IFF_UP) {
1161 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1162 if ((ifp->if_flags ^ adapter->if_flags) &
1163 (IFF_PROMISC | IFF_ALLMULTI)) {
1164 igb_disable_promisc(adapter);
1165 igb_set_promisc(adapter);
1168 igb_init_locked(adapter);
1170 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1172 adapter->if_flags = ifp->if_flags;
1173 IGB_CORE_UNLOCK(adapter);
1177 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1178 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1179 IGB_CORE_LOCK(adapter);
1180 igb_disable_intr(adapter);
1181 igb_set_multi(adapter);
1182 #ifdef DEVICE_POLLING
1183 if (!(ifp->if_capenable & IFCAP_POLLING))
1185 igb_enable_intr(adapter);
1186 IGB_CORE_UNLOCK(adapter);
1190 /* Check SOL/IDER usage */
1191 IGB_CORE_LOCK(adapter);
1192 if (e1000_check_reset_block(&adapter->hw)) {
1193 IGB_CORE_UNLOCK(adapter);
1194 device_printf(adapter->dev, "Media change is"
1195 " blocked due to SOL/IDER session.\n");
1198 IGB_CORE_UNLOCK(adapter);
1200 IOCTL_DEBUGOUT("ioctl rcv'd: \
1201 SIOCxIFMEDIA (Get/Set Interface Media)");
1202 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1208 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1210 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1211 #ifdef DEVICE_POLLING
1212 if (mask & IFCAP_POLLING) {
1213 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1214 error = ether_poll_register(igb_poll, ifp);
1217 IGB_CORE_LOCK(adapter);
1218 igb_disable_intr(adapter);
1219 ifp->if_capenable |= IFCAP_POLLING;
1220 IGB_CORE_UNLOCK(adapter);
1222 error = ether_poll_deregister(ifp);
1223 /* Enable interrupt even in error case */
1224 IGB_CORE_LOCK(adapter);
1225 igb_enable_intr(adapter);
1226 ifp->if_capenable &= ~IFCAP_POLLING;
1227 IGB_CORE_UNLOCK(adapter);
1231 if (mask & IFCAP_HWCSUM) {
1232 ifp->if_capenable ^= IFCAP_HWCSUM;
1235 if (mask & IFCAP_TSO4) {
1236 ifp->if_capenable ^= IFCAP_TSO4;
1239 if (mask & IFCAP_TSO6) {
1240 ifp->if_capenable ^= IFCAP_TSO6;
1243 if (mask & IFCAP_VLAN_HWTAGGING) {
1244 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1247 if (mask & IFCAP_VLAN_HWFILTER) {
1248 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1251 if (mask & IFCAP_VLAN_HWTSO) {
1252 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1255 if (mask & IFCAP_LRO) {
1256 ifp->if_capenable ^= IFCAP_LRO;
1259 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1261 VLAN_CAPABILITIES(ifp);
1266 error = ether_ioctl(ifp, command, data);
1274 /*********************************************************************
1277 * This routine is used in two ways. It is used by the stack as
1278 * init entry point in network interface structure. It is also used
1279 * by the driver as a hw/sw initialization routine to get to a
1282 * return 0 on success, positive on failure
1283 **********************************************************************/
1286 igb_init_locked(struct adapter *adapter)
1288 struct ifnet *ifp = adapter->ifp;
1289 device_t dev = adapter->dev;
1291 INIT_DEBUGOUT("igb_init: begin");
1293 IGB_CORE_LOCK_ASSERT(adapter);
1295 igb_disable_intr(adapter);
1296 callout_stop(&adapter->timer);
1298 /* Get the latest mac address, User can use a LAA */
1299 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1302 /* Put the address into the Receive Address Array */
1303 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1306 igb_update_link_status(adapter);
1308 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1310 /* Set hardware offload abilities */
1311 ifp->if_hwassist = 0;
1312 if (ifp->if_capenable & IFCAP_TXCSUM) {
1313 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1314 #if __FreeBSD_version >= 800000
1315 if (adapter->hw.mac.type == e1000_82576)
1316 ifp->if_hwassist |= CSUM_SCTP;
1320 if (ifp->if_capenable & IFCAP_TSO)
1321 ifp->if_hwassist |= CSUM_TSO;
1323 /* Configure for OS presence */
1324 igb_init_manageability(adapter);
1326 /* Prepare transmit descriptors and buffers */
1327 igb_setup_transmit_structures(adapter);
1328 igb_initialize_transmit_units(adapter);
1330 /* Setup Multicast table */
1331 igb_set_multi(adapter);
1334 ** Figure out the desired mbuf pool
1335 ** for doing jumbo/packetsplit
1337 if (adapter->max_frame_size <= 2048)
1338 adapter->rx_mbuf_sz = MCLBYTES;
1339 else if (adapter->max_frame_size <= 4096)
1340 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1342 adapter->rx_mbuf_sz = MJUM9BYTES;
1344 /* Prepare receive descriptors and buffers */
1345 if (igb_setup_receive_structures(adapter)) {
1346 device_printf(dev, "Could not setup receive structures\n");
1349 igb_initialize_receive_units(adapter);
1351 /* Enable VLAN support */
1352 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1353 igb_setup_vlan_hw_support(adapter);
1355 /* Don't lose promiscuous settings */
1356 igb_set_promisc(adapter);
1358 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1359 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1361 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1362 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1364 if (adapter->msix > 1) /* Set up queue routing */
1365 igb_configure_queues(adapter);
1367 /* this clears any pending interrupts */
1368 E1000_READ_REG(&adapter->hw, E1000_ICR);
1369 #ifdef DEVICE_POLLING
1371 * Only enable interrupts if we are not polling, make sure
1372 * they are off otherwise.
1374 if (ifp->if_capenable & IFCAP_POLLING)
1375 igb_disable_intr(adapter);
1377 #endif /* DEVICE_POLLING */
1379 igb_enable_intr(adapter);
1380 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1383 /* Set Energy Efficient Ethernet */
1384 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
1385 if (adapter->hw.mac.type == e1000_i354)
1386 e1000_set_eee_i354(&adapter->hw);
1388 e1000_set_eee_i350(&adapter->hw);
1395 struct adapter *adapter = arg;
1397 IGB_CORE_LOCK(adapter);
1398 igb_init_locked(adapter);
1399 IGB_CORE_UNLOCK(adapter);
1404 igb_handle_que(void *context, int pending)
1406 struct igb_queue *que = context;
1407 struct adapter *adapter = que->adapter;
1408 struct tx_ring *txr = que->txr;
1409 struct ifnet *ifp = adapter->ifp;
1411 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1414 more = igb_rxeof(que, adapter->rx_process_limit, NULL);
1418 #ifndef IGB_LEGACY_TX
1419 /* Process the stack queue only if not depleted */
1420 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1421 !drbr_empty(ifp, txr->br))
1422 igb_mq_start_locked(ifp, txr);
1424 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1425 igb_start_locked(txr, ifp);
1428 /* Do we need another? */
1430 taskqueue_enqueue(que->tq, &que->que_task);
1435 #ifdef DEVICE_POLLING
1436 if (ifp->if_capenable & IFCAP_POLLING)
1439 /* Reenable this interrupt */
1441 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1443 igb_enable_intr(adapter);
1446 /* Deal with link in a sleepable context */
1448 igb_handle_link(void *context, int pending)
1450 struct adapter *adapter = context;
1452 IGB_CORE_LOCK(adapter);
1453 igb_handle_link_locked(adapter);
1454 IGB_CORE_UNLOCK(adapter);
1458 igb_handle_link_locked(struct adapter *adapter)
1460 struct tx_ring *txr = adapter->tx_rings;
1461 struct ifnet *ifp = adapter->ifp;
1463 IGB_CORE_LOCK_ASSERT(adapter);
1464 adapter->hw.mac.get_link_status = 1;
1465 igb_update_link_status(adapter);
1466 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
1467 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1469 #ifndef IGB_LEGACY_TX
1470 /* Process the stack queue only if not depleted */
1471 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1472 !drbr_empty(ifp, txr->br))
1473 igb_mq_start_locked(ifp, txr);
1475 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1476 igb_start_locked(txr, ifp);
1483 /*********************************************************************
1485 * MSI/Legacy Deferred
1486 * Interrupt Service routine
1488 *********************************************************************/
1490 igb_irq_fast(void *arg)
1492 struct adapter *adapter = arg;
1493 struct igb_queue *que = adapter->queues;
1497 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1500 if (reg_icr == 0xffffffff)
1501 return FILTER_STRAY;
1503 /* Definitely not our interrupt. */
1505 return FILTER_STRAY;
1507 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1508 return FILTER_STRAY;
1511 * Mask interrupts until the taskqueue is finished running. This is
1512 * cheap, just assume that it is needed. This also works around the
1513 * MSI message reordering errata on certain systems.
1515 igb_disable_intr(adapter);
1516 taskqueue_enqueue(que->tq, &que->que_task);
1518 /* Link status change */
1519 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1520 taskqueue_enqueue(que->tq, &adapter->link_task);
1522 if (reg_icr & E1000_ICR_RXO)
1523 adapter->rx_overruns++;
1524 return FILTER_HANDLED;
1527 #ifdef DEVICE_POLLING
1528 #if __FreeBSD_version >= 800000
1529 #define POLL_RETURN_COUNT(a) (a)
1532 #define POLL_RETURN_COUNT(a)
1535 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1537 struct adapter *adapter = ifp->if_softc;
1538 struct igb_queue *que;
1539 struct tx_ring *txr;
1540 u32 reg_icr, rx_done = 0;
1541 u32 loop = IGB_MAX_LOOP;
1544 IGB_CORE_LOCK(adapter);
1545 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1546 IGB_CORE_UNLOCK(adapter);
1547 return POLL_RETURN_COUNT(rx_done);
1550 if (cmd == POLL_AND_CHECK_STATUS) {
1551 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1552 /* Link status change */
1553 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1554 igb_handle_link_locked(adapter);
1556 if (reg_icr & E1000_ICR_RXO)
1557 adapter->rx_overruns++;
1559 IGB_CORE_UNLOCK(adapter);
1561 for (int i = 0; i < adapter->num_queues; i++) {
1562 que = &adapter->queues[i];
1565 igb_rxeof(que, count, &rx_done);
1569 more = igb_txeof(txr);
1570 } while (loop-- && more);
1571 #ifndef IGB_LEGACY_TX
1572 if (!drbr_empty(ifp, txr->br))
1573 igb_mq_start_locked(ifp, txr);
1575 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1576 igb_start_locked(txr, ifp);
1581 return POLL_RETURN_COUNT(rx_done);
1583 #endif /* DEVICE_POLLING */
1585 /*********************************************************************
1587 * MSIX Que Interrupt Service routine
1589 **********************************************************************/
1591 igb_msix_que(void *arg)
1593 struct igb_queue *que = arg;
1594 struct adapter *adapter = que->adapter;
1595 struct ifnet *ifp = adapter->ifp;
1596 struct tx_ring *txr = que->txr;
1597 struct rx_ring *rxr = que->rxr;
1601 /* Ignore spurious interrupts */
1602 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1605 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1610 #ifndef IGB_LEGACY_TX
1611 /* Process the stack queue only if not depleted */
1612 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1613 !drbr_empty(ifp, txr->br))
1614 igb_mq_start_locked(ifp, txr);
1616 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1617 igb_start_locked(txr, ifp);
1621 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
1623 if (adapter->enable_aim == FALSE)
1626 ** Do Adaptive Interrupt Moderation:
1627 ** - Write out last calculated setting
1628 ** - Calculate based on average size over
1629 ** the last interval.
1631 if (que->eitr_setting)
1632 E1000_WRITE_REG(&adapter->hw,
1633 E1000_EITR(que->msix), que->eitr_setting);
1635 que->eitr_setting = 0;
1637 /* Idle, do nothing */
1638 if ((txr->bytes == 0) && (rxr->bytes == 0))
1641 /* Used half Default if sub-gig */
1642 if (adapter->link_speed != 1000)
1643 newitr = IGB_DEFAULT_ITR / 2;
1645 if ((txr->bytes) && (txr->packets))
1646 newitr = txr->bytes/txr->packets;
1647 if ((rxr->bytes) && (rxr->packets))
1648 newitr = max(newitr,
1649 (rxr->bytes / rxr->packets));
1650 newitr += 24; /* account for hardware frame, crc */
1651 /* set an upper boundary */
1652 newitr = min(newitr, 3000);
1653 /* Be nice to the mid range */
1654 if ((newitr > 300) && (newitr < 1200))
1655 newitr = (newitr / 3);
1657 newitr = (newitr / 2);
1659 newitr &= 0x7FFC; /* Mask invalid bits */
1660 if (adapter->hw.mac.type == e1000_82575)
1661 newitr |= newitr << 16;
1663 newitr |= E1000_EITR_CNT_IGNR;
1665 /* save for next interrupt */
1666 que->eitr_setting = newitr;
1675 /* Schedule a clean task if needed*/
1677 taskqueue_enqueue(que->tq, &que->que_task);
1679 /* Reenable this interrupt */
1680 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1685 /*********************************************************************
1687 * MSIX Link Interrupt Service routine
1689 **********************************************************************/
1692 igb_msix_link(void *arg)
1694 struct adapter *adapter = arg;
1697 ++adapter->link_irq;
1698 icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1699 if (!(icr & E1000_ICR_LSC))
1701 igb_handle_link(adapter, 0);
1705 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1706 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1711 /*********************************************************************
1713 * Media Ioctl callback
1715 * This routine is called whenever the user queries the status of
1716 * the interface using ifconfig.
1718 **********************************************************************/
1720 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1722 struct adapter *adapter = ifp->if_softc;
1724 INIT_DEBUGOUT("igb_media_status: begin");
1726 IGB_CORE_LOCK(adapter);
1727 igb_update_link_status(adapter);
1729 ifmr->ifm_status = IFM_AVALID;
1730 ifmr->ifm_active = IFM_ETHER;
1732 if (!adapter->link_active) {
1733 IGB_CORE_UNLOCK(adapter);
1737 ifmr->ifm_status |= IFM_ACTIVE;
1739 switch (adapter->link_speed) {
1741 ifmr->ifm_active |= IFM_10_T;
1745 ** Support for 100Mb SFP - these are Fiber
1746 ** but the media type appears as serdes
1748 if (adapter->hw.phy.media_type ==
1749 e1000_media_type_internal_serdes)
1750 ifmr->ifm_active |= IFM_100_FX;
1752 ifmr->ifm_active |= IFM_100_TX;
1755 ifmr->ifm_active |= IFM_1000_T;
1758 ifmr->ifm_active |= IFM_2500_SX;
1762 if (adapter->link_duplex == FULL_DUPLEX)
1763 ifmr->ifm_active |= IFM_FDX;
1765 ifmr->ifm_active |= IFM_HDX;
1767 IGB_CORE_UNLOCK(adapter);
1770 /*********************************************************************
1772 * Media Ioctl callback
1774 * This routine is called when the user changes speed/duplex using
1775 * media/mediopt option with ifconfig.
1777 **********************************************************************/
1779 igb_media_change(struct ifnet *ifp)
1781 struct adapter *adapter = ifp->if_softc;
1782 struct ifmedia *ifm = &adapter->media;
1784 INIT_DEBUGOUT("igb_media_change: begin");
1786 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1789 IGB_CORE_LOCK(adapter);
1790 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1792 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1793 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1798 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1799 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1802 adapter->hw.mac.autoneg = FALSE;
1803 adapter->hw.phy.autoneg_advertised = 0;
1804 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1805 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1807 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1810 adapter->hw.mac.autoneg = FALSE;
1811 adapter->hw.phy.autoneg_advertised = 0;
1812 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1813 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1815 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1818 device_printf(adapter->dev, "Unsupported media type\n");
1821 igb_init_locked(adapter);
1822 IGB_CORE_UNLOCK(adapter);
1828 /*********************************************************************
1830 * This routine maps the mbufs to Advanced TX descriptors.
1832 **********************************************************************/
1834 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1836 struct adapter *adapter = txr->adapter;
1837 u32 olinfo_status = 0, cmd_type_len;
1838 int i, j, error, nsegs;
1841 struct mbuf *m_head;
1842 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1844 struct igb_tx_buf *txbuf;
1845 union e1000_adv_tx_desc *txd = NULL;
1849 /* Basic descriptor defines */
1850 cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
1851 E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
1853 if (m_head->m_flags & M_VLANTAG)
1854 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1857 * Important to capture the first descriptor
1858 * used because it will contain the index of
1859 * the one we tell the hardware to report back
1861 first = txr->next_avail_desc;
1862 txbuf = &txr->tx_buffers[first];
1866 * Map the packet for DMA.
1869 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1870 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1872 if (__predict_false(error)) {
1877 /* Try it again? - one try */
1878 if (remap == TRUE) {
1880 m = m_defrag(*m_headp, M_NOWAIT);
1882 adapter->mbuf_defrag_failed++;
1892 txr->no_tx_dma_setup++;
1895 txr->no_tx_dma_setup++;
1902 /* Make certain there are enough descriptors */
1903 if (nsegs > txr->tx_avail - 2) {
1904 txr->no_desc_avail++;
1905 bus_dmamap_unload(txr->txtag, map);
1911 ** Set up the appropriate offload context
1912 ** this will consume the first descriptor
1914 error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1915 if (__predict_false(error)) {
1921 /* 82575 needs the queue index added */
1922 if (adapter->hw.mac.type == e1000_82575)
1923 olinfo_status |= txr->me << 4;
1925 i = txr->next_avail_desc;
1926 for (j = 0; j < nsegs; j++) {
1930 txbuf = &txr->tx_buffers[i];
1931 txd = &txr->tx_base[i];
1932 seglen = segs[j].ds_len;
1933 segaddr = htole64(segs[j].ds_addr);
1935 txd->read.buffer_addr = segaddr;
1936 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
1937 cmd_type_len | seglen);
1938 txd->read.olinfo_status = htole32(olinfo_status);
1940 if (++i == txr->num_desc)
1944 txd->read.cmd_type_len |=
1945 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1946 txr->tx_avail -= nsegs;
1947 txr->next_avail_desc = i;
1949 txbuf->m_head = m_head;
1951 ** Here we swap the map so the last descriptor,
1952 ** which gets the completion interrupt has the
1953 ** real map, and the first descriptor gets the
1954 ** unused map from this descriptor.
1956 txr->tx_buffers[first].map = txbuf->map;
1958 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1960 /* Set the EOP descriptor that will be marked done */
1961 txbuf = &txr->tx_buffers[first];
1964 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1965 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1967 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1968 * hardware that this frame is available to transmit.
1970 ++txr->total_packets;
1971 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1976 igb_set_promisc(struct adapter *adapter)
1978 struct ifnet *ifp = adapter->ifp;
1979 struct e1000_hw *hw = &adapter->hw;
1982 if (adapter->vf_ifp) {
1983 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
1987 reg = E1000_READ_REG(hw, E1000_RCTL);
1988 if (ifp->if_flags & IFF_PROMISC) {
1989 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1990 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1991 } else if (ifp->if_flags & IFF_ALLMULTI) {
1992 reg |= E1000_RCTL_MPE;
1993 reg &= ~E1000_RCTL_UPE;
1994 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1999 igb_disable_promisc(struct adapter *adapter)
2001 struct e1000_hw *hw = &adapter->hw;
2002 struct ifnet *ifp = adapter->ifp;
2006 if (adapter->vf_ifp) {
2007 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
2010 reg = E1000_READ_REG(hw, E1000_RCTL);
2011 reg &= (~E1000_RCTL_UPE);
2012 if (ifp->if_flags & IFF_ALLMULTI)
2013 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2015 struct ifmultiaddr *ifma;
2016 #if __FreeBSD_version < 800000
2019 if_maddr_rlock(ifp);
2021 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2022 if (ifma->ifma_addr->sa_family != AF_LINK)
2024 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2028 #if __FreeBSD_version < 800000
2029 IF_ADDR_UNLOCK(ifp);
2031 if_maddr_runlock(ifp);
2034 /* Don't disable if in MAX groups */
2035 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2036 reg &= (~E1000_RCTL_MPE);
2037 E1000_WRITE_REG(hw, E1000_RCTL, reg);
2041 /*********************************************************************
2044 * This routine is called whenever multicast address list is updated.
2046 **********************************************************************/
2049 igb_set_multi(struct adapter *adapter)
2051 struct ifnet *ifp = adapter->ifp;
2052 struct ifmultiaddr *ifma;
2058 IOCTL_DEBUGOUT("igb_set_multi: begin");
2061 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
2062 MAX_NUM_MULTICAST_ADDRESSES);
2064 #if __FreeBSD_version < 800000
2067 if_maddr_rlock(ifp);
2069 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2070 if (ifma->ifma_addr->sa_family != AF_LINK)
2073 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2076 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2077 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2080 #if __FreeBSD_version < 800000
2081 IF_ADDR_UNLOCK(ifp);
2083 if_maddr_runlock(ifp);
2086 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2087 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2088 reg_rctl |= E1000_RCTL_MPE;
2089 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2091 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2095 /*********************************************************************
2097 * This routine checks for link status,
2098 * updates statistics, and does the watchdog.
2100 **********************************************************************/
2103 igb_local_timer(void *arg)
2105 struct adapter *adapter = arg;
2106 device_t dev = adapter->dev;
2107 struct ifnet *ifp = adapter->ifp;
2108 struct tx_ring *txr = adapter->tx_rings;
2109 struct igb_queue *que = adapter->queues;
2110 int hung = 0, busy = 0;
2113 IGB_CORE_LOCK_ASSERT(adapter);
2115 igb_update_link_status(adapter);
2116 igb_update_stats_counters(adapter);
2119 ** Check the TX queues status
2120 ** - central locked handling of OACTIVE
2121 ** - watchdog only if all queues show hung
2123 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2124 if ((txr->queue_status & IGB_QUEUE_HUNG) &&
2125 (adapter->pause_frames == 0))
2127 if (txr->queue_status & IGB_QUEUE_DEPLETED)
2129 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
2130 taskqueue_enqueue(que->tq, &que->que_task);
2132 if (hung == adapter->num_queues)
2134 if (busy == adapter->num_queues)
2135 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2136 else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
2137 (busy < adapter->num_queues))
2138 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2140 adapter->pause_frames = 0;
2141 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
2142 #ifndef DEVICE_POLLING
2143 /* Schedule all queue interrupts - deadlock protection */
2144 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
2149 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2150 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2151 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2152 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2153 device_printf(dev,"TX(%d) desc avail = %d,"
2154 "Next TX to Clean = %d\n",
2155 txr->me, txr->tx_avail, txr->next_to_clean);
2156 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2157 adapter->watchdog_events++;
2158 igb_init_locked(adapter);
2162 igb_update_link_status(struct adapter *adapter)
2164 struct e1000_hw *hw = &adapter->hw;
2165 struct e1000_fc_info *fc = &hw->fc;
2166 struct ifnet *ifp = adapter->ifp;
2167 device_t dev = adapter->dev;
2168 struct tx_ring *txr = adapter->tx_rings;
2169 u32 link_check, thstat, ctrl;
2170 char *flowctl = NULL;
2172 link_check = thstat = ctrl = 0;
2174 /* Get the cached link value or read for real */
2175 switch (hw->phy.media_type) {
2176 case e1000_media_type_copper:
2177 if (hw->mac.get_link_status) {
2178 /* Do the work to read phy */
2179 e1000_check_for_link(hw);
2180 link_check = !hw->mac.get_link_status;
2184 case e1000_media_type_fiber:
2185 e1000_check_for_link(hw);
2186 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2189 case e1000_media_type_internal_serdes:
2190 e1000_check_for_link(hw);
2191 link_check = adapter->hw.mac.serdes_has_link;
2193 /* VF device is type_unknown */
2194 case e1000_media_type_unknown:
2195 e1000_check_for_link(hw);
2196 link_check = !hw->mac.get_link_status;
2202 /* Check for thermal downshift or shutdown */
2203 if (hw->mac.type == e1000_i350) {
2204 thstat = E1000_READ_REG(hw, E1000_THSTAT);
2205 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
2208 /* Get the flow control for display */
2209 switch (fc->current_mode) {
2210 case e1000_fc_rx_pause:
2213 case e1000_fc_tx_pause:
2225 /* Now we check if a transition has happened */
2226 if (link_check && (adapter->link_active == 0)) {
2227 e1000_get_speed_and_duplex(&adapter->hw,
2228 &adapter->link_speed, &adapter->link_duplex);
2230 device_printf(dev, "Link is up %d Mbps %s,"
2231 " Flow Control: %s\n",
2232 adapter->link_speed,
2233 ((adapter->link_duplex == FULL_DUPLEX) ?
2234 "Full Duplex" : "Half Duplex"), flowctl);
2235 adapter->link_active = 1;
2236 ifp->if_baudrate = adapter->link_speed * 1000000;
2237 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2238 (thstat & E1000_THSTAT_LINK_THROTTLE))
2239 device_printf(dev, "Link: thermal downshift\n");
2240 /* Delay Link Up for Phy update */
2241 if (((hw->mac.type == e1000_i210) ||
2242 (hw->mac.type == e1000_i211)) &&
2243 (hw->phy.id == I210_I_PHY_ID))
2244 msec_delay(I210_LINK_DELAY);
2245 /* Reset if the media type changed. */
2246 if (hw->dev_spec._82575.media_changed) {
2247 hw->dev_spec._82575.media_changed = false;
2248 adapter->flags |= IGB_MEDIA_RESET;
2251 /* This can sleep */
2252 if_link_state_change(ifp, LINK_STATE_UP);
2253 } else if (!link_check && (adapter->link_active == 1)) {
2254 ifp->if_baudrate = adapter->link_speed = 0;
2255 adapter->link_duplex = 0;
2257 device_printf(dev, "Link is Down\n");
2258 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2259 (thstat & E1000_THSTAT_PWR_DOWN))
2260 device_printf(dev, "Link: thermal shutdown\n");
2261 adapter->link_active = 0;
2262 /* This can sleep */
2263 if_link_state_change(ifp, LINK_STATE_DOWN);
2264 /* Reset queue state */
2265 for (int i = 0; i < adapter->num_queues; i++, txr++)
2266 txr->queue_status = IGB_QUEUE_IDLE;
2270 /*********************************************************************
2272 * This routine disables all traffic on the adapter by issuing a
2273 * global reset on the MAC and deallocates TX/RX buffers.
2275 **********************************************************************/
2280 struct adapter *adapter = arg;
2281 struct ifnet *ifp = adapter->ifp;
2282 struct tx_ring *txr = adapter->tx_rings;
2284 IGB_CORE_LOCK_ASSERT(adapter);
2286 INIT_DEBUGOUT("igb_stop: begin");
2288 igb_disable_intr(adapter);
2290 callout_stop(&adapter->timer);
2292 /* Tell the stack that the interface is no longer active */
2293 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2294 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2296 /* Disarm watchdog timer. */
2297 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2299 txr->queue_status = IGB_QUEUE_IDLE;
2303 e1000_reset_hw(&adapter->hw);
2304 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2306 e1000_led_off(&adapter->hw);
2307 e1000_cleanup_led(&adapter->hw);
2311 /*********************************************************************
2313 * Determine hardware revision.
2315 **********************************************************************/
2317 igb_identify_hardware(struct adapter *adapter)
2319 device_t dev = adapter->dev;
2321 /* Make sure our PCI config space has the necessary stuff set */
2322 pci_enable_busmaster(dev);
2323 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2325 /* Save off the information about this board */
2326 adapter->hw.vendor_id = pci_get_vendor(dev);
2327 adapter->hw.device_id = pci_get_device(dev);
2328 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2329 adapter->hw.subsystem_vendor_id =
2330 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2331 adapter->hw.subsystem_device_id =
2332 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2334 /* Set MAC type early for PCI setup */
2335 e1000_set_mac_type(&adapter->hw);
2337 /* Are we a VF device? */
2338 if ((adapter->hw.mac.type == e1000_vfadapt) ||
2339 (adapter->hw.mac.type == e1000_vfadapt_i350))
2340 adapter->vf_ifp = 1;
2342 adapter->vf_ifp = 0;
2346 igb_allocate_pci_resources(struct adapter *adapter)
2348 device_t dev = adapter->dev;
2352 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2354 if (adapter->pci_mem == NULL) {
2355 device_printf(dev, "Unable to allocate bus resource: memory\n");
2358 adapter->osdep.mem_bus_space_tag =
2359 rman_get_bustag(adapter->pci_mem);
2360 adapter->osdep.mem_bus_space_handle =
2361 rman_get_bushandle(adapter->pci_mem);
2362 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2364 adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2366 /* This will setup either MSI/X or MSI */
2367 adapter->msix = igb_setup_msix(adapter);
2368 adapter->hw.back = &adapter->osdep;
2373 /*********************************************************************
2375 * Setup the Legacy or MSI Interrupt handler
2377 **********************************************************************/
2379 igb_allocate_legacy(struct adapter *adapter)
2381 device_t dev = adapter->dev;
2382 struct igb_queue *que = adapter->queues;
2383 #ifndef IGB_LEGACY_TX
2384 struct tx_ring *txr = adapter->tx_rings;
2388 /* Turn off all interrupts */
2389 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2392 if (adapter->msix == 1)
2395 /* We allocate a single interrupt resource */
2396 adapter->res = bus_alloc_resource_any(dev,
2397 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2398 if (adapter->res == NULL) {
2399 device_printf(dev, "Unable to allocate bus resource: "
2404 #ifndef IGB_LEGACY_TX
2405 TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
2409 * Try allocating a fast interrupt and the associated deferred
2410 * processing contexts.
2412 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2413 /* Make tasklet for deferred link handling */
2414 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2415 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2416 taskqueue_thread_enqueue, &que->tq);
2417 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
2418 device_get_nameunit(adapter->dev));
2419 if ((error = bus_setup_intr(dev, adapter->res,
2420 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2421 adapter, &adapter->tag)) != 0) {
2422 device_printf(dev, "Failed to register fast interrupt "
2423 "handler: %d\n", error);
2424 taskqueue_free(que->tq);
2433 /*********************************************************************
2435 * Setup the MSIX Queue Interrupt handlers:
2437 **********************************************************************/
2439 igb_allocate_msix(struct adapter *adapter)
2441 device_t dev = adapter->dev;
2442 struct igb_queue *que = adapter->queues;
2443 int error, rid, vector = 0;
2445 /* Be sure to start with all interrupts disabled */
2446 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
2447 E1000_WRITE_FLUSH(&adapter->hw);
2449 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2451 que->res = bus_alloc_resource_any(dev,
2452 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2453 if (que->res == NULL) {
2455 "Unable to allocate bus resource: "
2456 "MSIX Queue Interrupt\n");
2459 error = bus_setup_intr(dev, que->res,
2460 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2461 igb_msix_que, que, &que->tag);
2464 device_printf(dev, "Failed to register Queue handler");
2467 #if __FreeBSD_version >= 800504
2468 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2471 if (adapter->hw.mac.type == e1000_82575)
2472 que->eims = E1000_EICR_TX_QUEUE0 << i;
2474 que->eims = 1 << vector;
2476 ** Bind the msix vector, and thus the
2477 ** rings to the corresponding cpu.
2479 if (adapter->num_queues > 1) {
2480 if (igb_last_bind_cpu < 0)
2481 igb_last_bind_cpu = CPU_FIRST();
2482 bus_bind_intr(dev, que->res, igb_last_bind_cpu);
2484 "Bound queue %d to cpu %d\n",
2485 i,igb_last_bind_cpu);
2486 igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
2488 #ifndef IGB_LEGACY_TX
2489 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
2492 /* Make tasklet for deferred handling */
2493 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2494 que->tq = taskqueue_create("igb_que", M_NOWAIT,
2495 taskqueue_thread_enqueue, &que->tq);
2496 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2497 device_get_nameunit(adapter->dev));
2502 adapter->res = bus_alloc_resource_any(dev,
2503 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2504 if (adapter->res == NULL) {
2506 "Unable to allocate bus resource: "
2507 "MSIX Link Interrupt\n");
2510 if ((error = bus_setup_intr(dev, adapter->res,
2511 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2512 igb_msix_link, adapter, &adapter->tag)) != 0) {
2513 device_printf(dev, "Failed to register Link handler");
2516 #if __FreeBSD_version >= 800504
2517 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2519 adapter->linkvec = vector;
2526 igb_configure_queues(struct adapter *adapter)
2528 struct e1000_hw *hw = &adapter->hw;
2529 struct igb_queue *que;
2530 u32 tmp, ivar = 0, newitr = 0;
2532 /* First turn on RSS capability */
2533 if (adapter->hw.mac.type != e1000_82575)
2534 E1000_WRITE_REG(hw, E1000_GPIE,
2535 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2536 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2539 switch (adapter->hw.mac.type) {
2546 case e1000_vfadapt_i350:
2548 for (int i = 0; i < adapter->num_queues; i++) {
2550 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2551 que = &adapter->queues[i];
2554 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2557 ivar |= que->msix | E1000_IVAR_VALID;
2559 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2562 for (int i = 0; i < adapter->num_queues; i++) {
2564 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2565 que = &adapter->queues[i];
2568 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2571 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2573 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2574 adapter->que_mask |= que->eims;
2577 /* And for the link interrupt */
2578 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2579 adapter->link_mask = 1 << adapter->linkvec;
2580 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2584 for (int i = 0; i < adapter->num_queues; i++) {
2585 u32 index = i & 0x7; /* Each IVAR has two entries */
2586 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2587 que = &adapter->queues[i];
2590 ivar |= que->msix | E1000_IVAR_VALID;
2593 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2595 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2596 adapter->que_mask |= que->eims;
2599 for (int i = 0; i < adapter->num_queues; i++) {
2600 u32 index = i & 0x7; /* Each IVAR has two entries */
2601 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2602 que = &adapter->queues[i];
2605 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2608 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2610 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2611 adapter->que_mask |= que->eims;
2614 /* And for the link interrupt */
2615 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2616 adapter->link_mask = 1 << adapter->linkvec;
2617 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2621 /* enable MSI-X support*/
2622 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2623 tmp |= E1000_CTRL_EXT_PBA_CLR;
2624 /* Auto-Mask interrupts upon ICR read. */
2625 tmp |= E1000_CTRL_EXT_EIAME;
2626 tmp |= E1000_CTRL_EXT_IRCA;
2627 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2630 for (int i = 0; i < adapter->num_queues; i++) {
2631 que = &adapter->queues[i];
2632 tmp = E1000_EICR_RX_QUEUE0 << i;
2633 tmp |= E1000_EICR_TX_QUEUE0 << i;
2635 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2637 adapter->que_mask |= que->eims;
2641 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2643 adapter->link_mask |= E1000_EIMS_OTHER;
2648 /* Set the starting interrupt rate */
2649 if (igb_max_interrupt_rate > 0)
2650 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
2652 if (hw->mac.type == e1000_82575)
2653 newitr |= newitr << 16;
2655 newitr |= E1000_EITR_CNT_IGNR;
2657 for (int i = 0; i < adapter->num_queues; i++) {
2658 que = &adapter->queues[i];
2659 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2667 igb_free_pci_resources(struct adapter *adapter)
2669 struct igb_queue *que = adapter->queues;
2670 device_t dev = adapter->dev;
2674 ** There is a slight possibility of a failure mode
2675 ** in attach that will result in entering this function
2676 ** before interrupt resources have been initialized, and
2677 ** in that case we do not want to execute the loops below
2678 ** We can detect this reliably by the state of the adapter
2681 if (adapter->res == NULL)
2685 * First release all the interrupt resources:
2687 for (int i = 0; i < adapter->num_queues; i++, que++) {
2688 rid = que->msix + 1;
2689 if (que->tag != NULL) {
2690 bus_teardown_intr(dev, que->res, que->tag);
2693 if (que->res != NULL)
2694 bus_release_resource(dev,
2695 SYS_RES_IRQ, rid, que->res);
2698 /* Clean the Legacy or Link interrupt last */
2699 if (adapter->linkvec) /* we are doing MSIX */
2700 rid = adapter->linkvec + 1;
2702 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2704 que = adapter->queues;
2705 if (adapter->tag != NULL) {
2706 taskqueue_drain(que->tq, &adapter->link_task);
2707 bus_teardown_intr(dev, adapter->res, adapter->tag);
2708 adapter->tag = NULL;
2710 if (adapter->res != NULL)
2711 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2713 for (int i = 0; i < adapter->num_queues; i++, que++) {
2714 if (que->tq != NULL) {
2715 #ifndef IGB_LEGACY_TX
2716 taskqueue_drain(que->tq, &que->txr->txq_task);
2718 taskqueue_drain(que->tq, &que->que_task);
2719 taskqueue_free(que->tq);
2724 pci_release_msi(dev);
2726 if (adapter->msix_mem != NULL)
2727 bus_release_resource(dev, SYS_RES_MEMORY,
2728 adapter->memrid, adapter->msix_mem);
2730 if (adapter->pci_mem != NULL)
2731 bus_release_resource(dev, SYS_RES_MEMORY,
2732 PCIR_BAR(0), adapter->pci_mem);
2737 * Setup Either MSI/X or MSI
2740 igb_setup_msix(struct adapter *adapter)
2742 device_t dev = adapter->dev;
2743 int bar, want, queues, msgs, maxqueues;
2745 /* tuneable override */
2746 if (igb_enable_msix == 0)
2749 /* First try MSI/X */
2750 msgs = pci_msix_count(dev);
2754 ** Some new devices, as with ixgbe, now may
2755 ** use a different BAR, so we need to keep
2756 ** track of which is used.
2758 adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
2759 bar = pci_read_config(dev, adapter->memrid, 4);
2760 if (bar == 0) /* use next bar */
2761 adapter->memrid += 4;
2762 adapter->msix_mem = bus_alloc_resource_any(dev,
2763 SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
2764 if (adapter->msix_mem == NULL) {
2765 /* May not be enabled */
2766 device_printf(adapter->dev,
2767 "Unable to map MSIX table \n");
2771 /* Figure out a reasonable auto config value */
2772 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2774 /* Manual override */
2775 if (igb_num_queues != 0)
2776 queues = igb_num_queues;
2778 /* Sanity check based on HW */
2779 switch (adapter->hw.mac.type) {
2795 default: /* VF interfaces */
2799 if (queues > maxqueues)
2802 /* Manual override */
2803 if (igb_num_queues != 0)
2804 queues = igb_num_queues;
2807 ** One vector (RX/TX pair) per queue
2808 ** plus an additional for Link interrupt
2814 device_printf(adapter->dev,
2815 "MSIX Configuration Problem, "
2816 "%d vectors configured, but %d queues wanted!\n",
2820 if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2821 device_printf(adapter->dev,
2822 "Using MSIX interrupts with %d vectors\n", msgs);
2823 adapter->num_queues = queues;
2827 ** If MSIX alloc failed or provided us with
2828 ** less than needed, free and fall through to MSI
2830 pci_release_msi(dev);
2833 if (adapter->msix_mem != NULL) {
2834 bus_release_resource(dev, SYS_RES_MEMORY,
2835 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2836 adapter->msix_mem = NULL;
2839 if (pci_alloc_msi(dev, &msgs) == 0) {
2840 device_printf(adapter->dev," Using an MSI interrupt\n");
2843 device_printf(adapter->dev," Using a Legacy interrupt\n");
2847 /*********************************************************************
2849 * Initialize the DMA Coalescing feature
2851 **********************************************************************/
2853 igb_init_dmac(struct adapter *adapter, u32 pba)
2855 device_t dev = adapter->dev;
2856 struct e1000_hw *hw = &adapter->hw;
2857 u32 dmac, reg = ~E1000_DMACR_DMAC_EN;
2860 if (hw->mac.type == e1000_i211)
2863 if (hw->mac.type > e1000_82580) {
2865 if (adapter->dmac == 0) { /* Disabling it */
2866 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2869 device_printf(dev, "DMA Coalescing enabled\n");
2871 /* Set starting threshold */
2872 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
2874 hwm = 64 * pba - adapter->max_frame_size / 16;
2875 if (hwm < 64 * (pba - 6))
2876 hwm = 64 * (pba - 6);
2877 reg = E1000_READ_REG(hw, E1000_FCRTC);
2878 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
2879 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
2880 & E1000_FCRTC_RTH_COAL_MASK);
2881 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
2884 dmac = pba - adapter->max_frame_size / 512;
2885 if (dmac < pba - 10)
2887 reg = E1000_READ_REG(hw, E1000_DMACR);
2888 reg &= ~E1000_DMACR_DMACTHR_MASK;
2889 reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
2890 & E1000_DMACR_DMACTHR_MASK);
2892 /* transition to L0x or L1 if available..*/
2893 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
2895 /* Check if status is 2.5Gb backplane connection
2896 * before configuration of watchdog timer, which is
2897 * in msec values in 12.8usec intervals
2898 * watchdog timer= msec values in 32usec intervals
2899 * for non 2.5Gb connection
2901 if (hw->mac.type == e1000_i354) {
2902 int status = E1000_READ_REG(hw, E1000_STATUS);
2903 if ((status & E1000_STATUS_2P5_SKU) &&
2904 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2905 reg |= ((adapter->dmac * 5) >> 6);
2907 reg |= (adapter->dmac >> 5);
2909 reg |= (adapter->dmac >> 5);
2912 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2914 #ifdef I210_OBFF_SUPPORT
2916 * Set the OBFF Rx threshold to DMA Coalescing Rx
2917 * threshold - 2KB and enable the feature in the
2918 * hardware for I210.
2920 if (hw->mac.type == e1000_i210) {
2921 int obff = dmac - 2;
2922 reg = E1000_READ_REG(hw, E1000_DOBFFCTL);
2923 reg &= ~E1000_DOBFFCTL_OBFFTHR_MASK;
2924 reg |= (obff & E1000_DOBFFCTL_OBFFTHR_MASK)
2925 | E1000_DOBFFCTL_EXIT_ACT_MASK;
2926 E1000_WRITE_REG(hw, E1000_DOBFFCTL, reg);
2929 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
2931 /* Set the interval before transition */
2932 reg = E1000_READ_REG(hw, E1000_DMCTLX);
2933 if (hw->mac.type == e1000_i350)
2934 reg |= IGB_DMCTLX_DCFLUSH_DIS;
2936 ** in 2.5Gb connection, TTLX unit is 0.4 usec
2937 ** which is 0x4*2 = 0xA. But delay is still 4 usec
2939 if (hw->mac.type == e1000_i354) {
2940 int status = E1000_READ_REG(hw, E1000_STATUS);
2941 if ((status & E1000_STATUS_2P5_SKU) &&
2942 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2950 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
2952 /* free space in tx packet buffer to wake from DMA coal */
2953 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
2954 (2 * adapter->max_frame_size)) >> 6);
2956 /* make low power state decision controlled by DMA coal */
2957 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2958 reg &= ~E1000_PCIEMISC_LX_DECISION;
2959 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
2961 } else if (hw->mac.type == e1000_82580) {
2962 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2963 E1000_WRITE_REG(hw, E1000_PCIEMISC,
2964 reg & ~E1000_PCIEMISC_LX_DECISION);
2965 E1000_WRITE_REG(hw, E1000_DMACR, 0);
2970 /*********************************************************************
2972 * Set up an fresh starting state
2974 **********************************************************************/
2976 igb_reset(struct adapter *adapter)
2978 device_t dev = adapter->dev;
2979 struct e1000_hw *hw = &adapter->hw;
2980 struct e1000_fc_info *fc = &hw->fc;
2981 struct ifnet *ifp = adapter->ifp;
2985 INIT_DEBUGOUT("igb_reset: begin");
2987 /* Let the firmware know the OS is in control */
2988 igb_get_hw_control(adapter);
2991 * Packet Buffer Allocation (PBA)
2992 * Writing PBA sets the receive portion of the buffer
2993 * the remainder is used for the transmit buffer.
2995 switch (hw->mac.type) {
2997 pba = E1000_PBA_32K;
3001 pba = E1000_READ_REG(hw, E1000_RXPBS);
3002 pba &= E1000_RXPBS_SIZE_MASK_82576;
3007 case e1000_vfadapt_i350:
3008 pba = E1000_READ_REG(hw, E1000_RXPBS);
3009 pba = e1000_rxpbs_adjust_82580(pba);
3013 pba = E1000_PBA_34K;
3018 /* Special needs in case of Jumbo frames */
3019 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
3020 u32 tx_space, min_tx, min_rx;
3021 pba = E1000_READ_REG(hw, E1000_PBA);
3022 tx_space = pba >> 16;
3024 min_tx = (adapter->max_frame_size +
3025 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
3026 min_tx = roundup2(min_tx, 1024);
3028 min_rx = adapter->max_frame_size;
3029 min_rx = roundup2(min_rx, 1024);
3031 if (tx_space < min_tx &&
3032 ((min_tx - tx_space) < pba)) {
3033 pba = pba - (min_tx - tx_space);
3035 * if short on rx space, rx wins
3036 * and must trump tx adjustment
3041 E1000_WRITE_REG(hw, E1000_PBA, pba);
3044 INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
3047 * These parameters control the automatic generation (Tx) and
3048 * response (Rx) to Ethernet PAUSE frames.
3049 * - High water mark should allow for at least two frames to be
3050 * received after sending an XOFF.
3051 * - Low water mark works best when it is very near the high water mark.
3052 * This allows the receiver to restart by sending XON when it has
3055 hwm = min(((pba << 10) * 9 / 10),
3056 ((pba << 10) - 2 * adapter->max_frame_size));
3058 if (hw->mac.type < e1000_82576) {
3059 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
3060 fc->low_water = fc->high_water - 8;
3062 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
3063 fc->low_water = fc->high_water - 16;
3066 fc->pause_time = IGB_FC_PAUSE_TIME;
3067 fc->send_xon = TRUE;
3069 fc->requested_mode = adapter->fc;
3071 fc->requested_mode = e1000_fc_default;
3073 /* Issue a global reset */
3075 E1000_WRITE_REG(hw, E1000_WUC, 0);
3077 /* Reset for AutoMediaDetect */
3078 if (adapter->flags & IGB_MEDIA_RESET) {
3079 e1000_setup_init_funcs(hw, TRUE);
3080 e1000_get_bus_info(hw);
3081 adapter->flags &= ~IGB_MEDIA_RESET;
3084 if (e1000_init_hw(hw) < 0)
3085 device_printf(dev, "Hardware Initialization Failed\n");
3087 /* Setup DMA Coalescing */
3088 igb_init_dmac(adapter, pba);
3090 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
3091 e1000_get_phy_info(hw);
3092 e1000_check_for_link(hw);
3096 /*********************************************************************
3098 * Setup networking device structure and register an interface.
3100 **********************************************************************/
3102 igb_setup_interface(device_t dev, struct adapter *adapter)
3106 INIT_DEBUGOUT("igb_setup_interface: begin");
3108 ifp = adapter->ifp = if_alloc(IFT_ETHER);
3110 device_printf(dev, "can not allocate ifnet structure\n");
3113 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3114 ifp->if_init = igb_init;
3115 ifp->if_softc = adapter;
3116 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3117 ifp->if_ioctl = igb_ioctl;
3118 #ifndef IGB_LEGACY_TX
3119 ifp->if_transmit = igb_mq_start;
3120 ifp->if_qflush = igb_qflush;
3122 ifp->if_start = igb_start;
3123 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3124 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3125 IFQ_SET_READY(&ifp->if_snd);
3128 ether_ifattach(ifp, adapter->hw.mac.addr);
3130 ifp->if_capabilities = ifp->if_capenable = 0;
3132 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3133 ifp->if_capabilities |= IFCAP_TSO;
3134 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
3135 ifp->if_capenable = ifp->if_capabilities;
3137 /* Don't enable LRO by default */
3138 ifp->if_capabilities |= IFCAP_LRO;
3140 #ifdef DEVICE_POLLING
3141 ifp->if_capabilities |= IFCAP_POLLING;
3145 * Tell the upper layer(s) we
3146 * support full VLAN capability.
3148 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3149 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
3152 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
3157 ** Don't turn this on by default, if vlans are
3158 ** created on another pseudo device (eg. lagg)
3159 ** then vlan events are not passed thru, breaking
3160 ** operation, but with HW FILTER off it works. If
3161 ** using vlans directly on the igb driver you can
3162 ** enable this and get full hardware tag filtering.
3164 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3167 * Specify the media types supported by this adapter and register
3168 * callbacks to update media and link information
3170 ifmedia_init(&adapter->media, IFM_IMASK,
3171 igb_media_change, igb_media_status);
3172 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3173 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3174 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
3176 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
3178 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3179 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3181 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3183 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3185 if (adapter->hw.phy.type != e1000_phy_ife) {
3186 ifmedia_add(&adapter->media,
3187 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3188 ifmedia_add(&adapter->media,
3189 IFM_ETHER | IFM_1000_T, 0, NULL);
3192 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3193 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3199 * Manage DMA'able memory.
3202 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3206 *(bus_addr_t *) arg = segs[0].ds_addr;
3210 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
3211 struct igb_dma_alloc *dma, int mapflags)
3215 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3216 IGB_DBA_ALIGN, 0, /* alignment, bounds */
3217 BUS_SPACE_MAXADDR, /* lowaddr */
3218 BUS_SPACE_MAXADDR, /* highaddr */
3219 NULL, NULL, /* filter, filterarg */
3222 size, /* maxsegsize */
3224 NULL, /* lockfunc */
3228 device_printf(adapter->dev,
3229 "%s: bus_dma_tag_create failed: %d\n",
3234 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3235 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3237 device_printf(adapter->dev,
3238 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3239 __func__, (uintmax_t)size, error);
3244 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3245 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3246 if (error || dma->dma_paddr == 0) {
3247 device_printf(adapter->dev,
3248 "%s: bus_dmamap_load failed: %d\n",
3256 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3258 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3259 bus_dma_tag_destroy(dma->dma_tag);
3261 dma->dma_map = NULL;
3262 dma->dma_tag = NULL;
3268 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
3270 if (dma->dma_tag == NULL)
3272 if (dma->dma_map != NULL) {
3273 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3274 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3275 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3276 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3277 dma->dma_map = NULL;
3279 bus_dma_tag_destroy(dma->dma_tag);
3280 dma->dma_tag = NULL;
3284 /*********************************************************************
3286 * Allocate memory for the transmit and receive rings, and then
3287 * the descriptors associated with each, called only once at attach.
3289 **********************************************************************/
3291 igb_allocate_queues(struct adapter *adapter)
3293 device_t dev = adapter->dev;
3294 struct igb_queue *que = NULL;
3295 struct tx_ring *txr = NULL;
3296 struct rx_ring *rxr = NULL;
3297 int rsize, tsize, error = E1000_SUCCESS;
3298 int txconf = 0, rxconf = 0;
3300 /* First allocate the top level queue structs */
3301 if (!(adapter->queues =
3302 (struct igb_queue *) malloc(sizeof(struct igb_queue) *
3303 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3304 device_printf(dev, "Unable to allocate queue memory\n");
3309 /* Next allocate the TX ring struct memory */
3310 if (!(adapter->tx_rings =
3311 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3312 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3313 device_printf(dev, "Unable to allocate TX ring memory\n");
3318 /* Now allocate the RX */
3319 if (!(adapter->rx_rings =
3320 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3321 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3322 device_printf(dev, "Unable to allocate RX ring memory\n");
3327 tsize = roundup2(adapter->num_tx_desc *
3328 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
3330 * Now set up the TX queues, txconf is needed to handle the
3331 * possibility that things fail midcourse and we need to
3332 * undo memory gracefully
3334 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3335 /* Set up some basics */
3336 txr = &adapter->tx_rings[i];
3337 txr->adapter = adapter;
3339 txr->num_desc = adapter->num_tx_desc;
3341 /* Initialize the TX lock */
3342 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3343 device_get_nameunit(dev), txr->me);
3344 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3346 if (igb_dma_malloc(adapter, tsize,
3347 &txr->txdma, BUS_DMA_NOWAIT)) {
3349 "Unable to allocate TX Descriptor memory\n");
3353 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
3354 bzero((void *)txr->tx_base, tsize);
3356 /* Now allocate transmit buffers for the ring */
3357 if (igb_allocate_transmit_buffers(txr)) {
3359 "Critical Failure setting up transmit buffers\n");
3363 #ifndef IGB_LEGACY_TX
3364 /* Allocate a buf ring */
3365 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
3366 M_WAITOK, &txr->tx_mtx);
3371 * Next the RX queues...
3373 rsize = roundup2(adapter->num_rx_desc *
3374 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3375 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3376 rxr = &adapter->rx_rings[i];
3377 rxr->adapter = adapter;
3380 /* Initialize the RX lock */
3381 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3382 device_get_nameunit(dev), txr->me);
3383 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3385 if (igb_dma_malloc(adapter, rsize,
3386 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3388 "Unable to allocate RxDescriptor memory\n");
3392 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3393 bzero((void *)rxr->rx_base, rsize);
3395 /* Allocate receive buffers for the ring*/
3396 if (igb_allocate_receive_buffers(rxr)) {
3398 "Critical Failure setting up receive buffers\n");
3405 ** Finally set up the queue holding structs
3407 for (int i = 0; i < adapter->num_queues; i++) {
3408 que = &adapter->queues[i];
3409 que->adapter = adapter;
3410 que->txr = &adapter->tx_rings[i];
3411 que->rxr = &adapter->rx_rings[i];
3417 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3418 igb_dma_free(adapter, &rxr->rxdma);
3420 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3421 igb_dma_free(adapter, &txr->txdma);
3422 free(adapter->rx_rings, M_DEVBUF);
3424 #ifndef IGB_LEGACY_TX
3425 buf_ring_free(txr->br, M_DEVBUF);
3427 free(adapter->tx_rings, M_DEVBUF);
3429 free(adapter->queues, M_DEVBUF);
3434 /*********************************************************************
3436 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3437 * the information needed to transmit a packet on the wire. This is
3438 * called only once at attach, setup is done every reset.
3440 **********************************************************************/
3442 igb_allocate_transmit_buffers(struct tx_ring *txr)
3444 struct adapter *adapter = txr->adapter;
3445 device_t dev = adapter->dev;
3446 struct igb_tx_buf *txbuf;
3450 * Setup DMA descriptor areas.
3452 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3453 1, 0, /* alignment, bounds */
3454 BUS_SPACE_MAXADDR, /* lowaddr */
3455 BUS_SPACE_MAXADDR, /* highaddr */
3456 NULL, NULL, /* filter, filterarg */
3457 IGB_TSO_SIZE, /* maxsize */
3458 IGB_MAX_SCATTER, /* nsegments */
3459 PAGE_SIZE, /* maxsegsize */
3461 NULL, /* lockfunc */
3462 NULL, /* lockfuncarg */
3464 device_printf(dev,"Unable to allocate TX DMA tag\n");
3468 if (!(txr->tx_buffers =
3469 (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
3470 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3471 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3476 /* Create the descriptor buffer dma maps */
3477 txbuf = txr->tx_buffers;
3478 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3479 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3481 device_printf(dev, "Unable to create TX DMA map\n");
3488 /* We free all, it handles case where we are in the middle */
3489 igb_free_transmit_structures(adapter);
3493 /*********************************************************************
3495 * Initialize a transmit ring.
3497 **********************************************************************/
3499 igb_setup_transmit_ring(struct tx_ring *txr)
3501 struct adapter *adapter = txr->adapter;
3502 struct igb_tx_buf *txbuf;
3505 struct netmap_adapter *na = NA(adapter->ifp);
3506 struct netmap_slot *slot;
3507 #endif /* DEV_NETMAP */
3509 /* Clear the old descriptor contents */
3512 slot = netmap_reset(na, NR_TX, txr->me, 0);
3513 #endif /* DEV_NETMAP */
3514 bzero((void *)txr->tx_base,
3515 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3517 txr->next_avail_desc = 0;
3518 txr->next_to_clean = 0;
3520 /* Free any existing tx buffers. */
3521 txbuf = txr->tx_buffers;
3522 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3523 if (txbuf->m_head != NULL) {
3524 bus_dmamap_sync(txr->txtag, txbuf->map,
3525 BUS_DMASYNC_POSTWRITE);
3526 bus_dmamap_unload(txr->txtag, txbuf->map);
3527 m_freem(txbuf->m_head);
3528 txbuf->m_head = NULL;
3532 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3533 /* no need to set the address */
3534 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3536 #endif /* DEV_NETMAP */
3537 /* clear the watch index */
3541 /* Set number of descriptors available */
3542 txr->tx_avail = adapter->num_tx_desc;
3544 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3545 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3549 /*********************************************************************
3551 * Initialize all transmit rings.
3553 **********************************************************************/
3555 igb_setup_transmit_structures(struct adapter *adapter)
3557 struct tx_ring *txr = adapter->tx_rings;
3559 for (int i = 0; i < adapter->num_queues; i++, txr++)
3560 igb_setup_transmit_ring(txr);
3565 /*********************************************************************
3567 * Enable transmit unit.
3569 **********************************************************************/
3571 igb_initialize_transmit_units(struct adapter *adapter)
3573 struct tx_ring *txr = adapter->tx_rings;
3574 struct e1000_hw *hw = &adapter->hw;
3577 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3580 /* Setup the Tx Descriptor Rings */
3581 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3582 u64 bus_addr = txr->txdma.dma_paddr;
3584 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3585 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3586 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3587 (uint32_t)(bus_addr >> 32));
3588 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3589 (uint32_t)bus_addr);
3591 /* Setup the HW Tx Head and Tail descriptor pointers */
3592 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3593 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3595 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3596 E1000_READ_REG(hw, E1000_TDBAL(i)),
3597 E1000_READ_REG(hw, E1000_TDLEN(i)));
3599 txr->queue_status = IGB_QUEUE_IDLE;
3601 txdctl |= IGB_TX_PTHRESH;
3602 txdctl |= IGB_TX_HTHRESH << 8;
3603 txdctl |= IGB_TX_WTHRESH << 16;
3604 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3605 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3608 if (adapter->vf_ifp)
3611 e1000_config_collision_dist(hw);
3613 /* Program the Transmit Control Register */
3614 tctl = E1000_READ_REG(hw, E1000_TCTL);
3615 tctl &= ~E1000_TCTL_CT;
3616 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3617 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3619 /* This write will effectively turn on the transmit unit. */
3620 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3623 /*********************************************************************
3625 * Free all transmit rings.
3627 **********************************************************************/
3629 igb_free_transmit_structures(struct adapter *adapter)
3631 struct tx_ring *txr = adapter->tx_rings;
3633 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3635 igb_free_transmit_buffers(txr);
3636 igb_dma_free(adapter, &txr->txdma);
3638 IGB_TX_LOCK_DESTROY(txr);
3640 free(adapter->tx_rings, M_DEVBUF);
3643 /*********************************************************************
3645 * Free transmit ring related data structures.
3647 **********************************************************************/
3649 igb_free_transmit_buffers(struct tx_ring *txr)
3651 struct adapter *adapter = txr->adapter;
3652 struct igb_tx_buf *tx_buffer;
3655 INIT_DEBUGOUT("free_transmit_ring: begin");
3657 if (txr->tx_buffers == NULL)
3660 tx_buffer = txr->tx_buffers;
3661 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3662 if (tx_buffer->m_head != NULL) {
3663 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3664 BUS_DMASYNC_POSTWRITE);
3665 bus_dmamap_unload(txr->txtag,
3667 m_freem(tx_buffer->m_head);
3668 tx_buffer->m_head = NULL;
3669 if (tx_buffer->map != NULL) {
3670 bus_dmamap_destroy(txr->txtag,
3672 tx_buffer->map = NULL;
3674 } else if (tx_buffer->map != NULL) {
3675 bus_dmamap_unload(txr->txtag,
3677 bus_dmamap_destroy(txr->txtag,
3679 tx_buffer->map = NULL;
3682 #ifndef IGB_LEGACY_TX
3683 if (txr->br != NULL)
3684 buf_ring_free(txr->br, M_DEVBUF);
3686 if (txr->tx_buffers != NULL) {
3687 free(txr->tx_buffers, M_DEVBUF);
3688 txr->tx_buffers = NULL;
3690 if (txr->txtag != NULL) {
3691 bus_dma_tag_destroy(txr->txtag);
3697 /**********************************************************************
3699 * Setup work for hardware segmentation offload (TSO) on
3700 * adapters using advanced tx descriptors
3702 **********************************************************************/
3704 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3705 u32 *cmd_type_len, u32 *olinfo_status)
3707 struct adapter *adapter = txr->adapter;
3708 struct e1000_adv_tx_context_desc *TXD;
3709 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3710 u32 mss_l4len_idx = 0, paylen;
3711 u16 vtag = 0, eh_type;
3712 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3713 struct ether_vlan_header *eh;
3715 struct ip6_hdr *ip6;
3724 * Determine where frame payload starts.
3725 * Jump over vlan headers if already present
3727 eh = mtod(mp, struct ether_vlan_header *);
3728 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3729 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3730 eh_type = eh->evl_proto;
3732 ehdrlen = ETHER_HDR_LEN;
3733 eh_type = eh->evl_encap_proto;
3736 switch (ntohs(eh_type)) {
3738 case ETHERTYPE_IPV6:
3739 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3740 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3741 if (ip6->ip6_nxt != IPPROTO_TCP)
3743 ip_hlen = sizeof(struct ip6_hdr);
3744 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3745 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3746 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3747 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3752 ip = (struct ip *)(mp->m_data + ehdrlen);
3753 if (ip->ip_p != IPPROTO_TCP)
3756 ip_hlen = ip->ip_hl << 2;
3757 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3758 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3759 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3760 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3761 /* Tell transmit desc to also do IPv4 checksum. */
3762 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3766 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3767 __func__, ntohs(eh_type));
3771 ctxd = txr->next_avail_desc;
3772 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3774 tcp_hlen = th->th_off << 2;
3776 /* This is used in the transmit desc in encap */
3777 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3779 /* VLAN MACLEN IPLEN */
3780 if (mp->m_flags & M_VLANTAG) {
3781 vtag = htole16(mp->m_pkthdr.ether_vtag);
3782 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3785 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3786 vlan_macip_lens |= ip_hlen;
3787 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3789 /* ADV DTYPE TUCMD */
3790 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3791 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3792 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3795 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3796 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3797 /* 82575 needs the queue index added */
3798 if (adapter->hw.mac.type == e1000_82575)
3799 mss_l4len_idx |= txr->me << 4;
3800 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3802 TXD->seqnum_seed = htole32(0);
3804 if (++ctxd == txr->num_desc)
3808 txr->next_avail_desc = ctxd;
3809 *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3810 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3811 *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
3816 /*********************************************************************
3818 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3820 **********************************************************************/
3823 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3824 u32 *cmd_type_len, u32 *olinfo_status)
3826 struct e1000_adv_tx_context_desc *TXD;
3827 struct adapter *adapter = txr->adapter;
3828 struct ether_vlan_header *eh;
3830 struct ip6_hdr *ip6;
3831 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
3832 int ehdrlen, ip_hlen = 0;
3836 int ctxd = txr->next_avail_desc;
3839 /* First check if TSO is to be used */
3840 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3841 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3843 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3846 /* Indicate the whole packet as payload when not doing TSO */
3847 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
3849 /* Now ready a context descriptor */
3850 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3853 ** In advanced descriptors the vlan tag must
3854 ** be placed into the context descriptor. Hence
3855 ** we need to make one even if not doing offloads.
3857 if (mp->m_flags & M_VLANTAG) {
3858 vtag = htole16(mp->m_pkthdr.ether_vtag);
3859 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3860 } else if (offload == FALSE) /* ... no offload to do */
3864 * Determine where frame payload starts.
3865 * Jump over vlan headers if already present,
3866 * helpful for QinQ too.
3868 eh = mtod(mp, struct ether_vlan_header *);
3869 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3870 etype = ntohs(eh->evl_proto);
3871 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3873 etype = ntohs(eh->evl_encap_proto);
3874 ehdrlen = ETHER_HDR_LEN;
3877 /* Set the ether header length */
3878 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3882 ip = (struct ip *)(mp->m_data + ehdrlen);
3883 ip_hlen = ip->ip_hl << 2;
3885 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3887 case ETHERTYPE_IPV6:
3888 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3889 ip_hlen = sizeof(struct ip6_hdr);
3890 /* XXX-BZ this will go badly in case of ext hdrs. */
3891 ipproto = ip6->ip6_nxt;
3892 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3899 vlan_macip_lens |= ip_hlen;
3900 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3904 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3905 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3908 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3909 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3912 #if __FreeBSD_version >= 800000
3914 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3915 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3923 if (offload) /* For the TX descriptor setup */
3924 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3926 /* 82575 needs the queue index added */
3927 if (adapter->hw.mac.type == e1000_82575)
3928 mss_l4len_idx = txr->me << 4;
3930 /* Now copy bits into descriptor */
3931 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3932 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3933 TXD->seqnum_seed = htole32(0);
3934 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3936 /* We've consumed the first desc, adjust counters */
3937 if (++ctxd == txr->num_desc)
3939 txr->next_avail_desc = ctxd;
3945 /**********************************************************************
3947 * Examine each tx_buffer in the used queue. If the hardware is done
3948 * processing the packet then free associated resources. The
3949 * tx_buffer is put back on the free queue.
3951 * TRUE return means there's work in the ring to clean, FALSE its empty.
3952 **********************************************************************/
3954 igb_txeof(struct tx_ring *txr)
3956 struct adapter *adapter = txr->adapter;
3957 struct ifnet *ifp = adapter->ifp;
3958 u32 work, processed = 0;
3959 u16 limit = txr->process_limit;
3960 struct igb_tx_buf *buf;
3961 union e1000_adv_tx_desc *txd;
3963 mtx_assert(&txr->tx_mtx, MA_OWNED);
3966 if (netmap_tx_irq(ifp, txr->me))
3968 #endif /* DEV_NETMAP */
3970 if (txr->tx_avail == txr->num_desc) {
3971 txr->queue_status = IGB_QUEUE_IDLE;
3975 /* Get work starting point */
3976 work = txr->next_to_clean;
3977 buf = &txr->tx_buffers[work];
3978 txd = &txr->tx_base[work];
3979 work -= txr->num_desc; /* The distance to ring end */
3980 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3981 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3983 union e1000_adv_tx_desc *eop = buf->eop;
3984 if (eop == NULL) /* No work */
3987 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
3988 break; /* I/O not complete */
3992 buf->m_head->m_pkthdr.len;
3993 bus_dmamap_sync(txr->txtag,
3995 BUS_DMASYNC_POSTWRITE);
3996 bus_dmamap_unload(txr->txtag,
3998 m_freem(buf->m_head);
4004 /* We clean the range if multi segment */
4005 while (txd != eop) {
4009 /* wrap the ring? */
4010 if (__predict_false(!work)) {
4011 work -= txr->num_desc;
4012 buf = txr->tx_buffers;
4017 buf->m_head->m_pkthdr.len;
4018 bus_dmamap_sync(txr->txtag,
4020 BUS_DMASYNC_POSTWRITE);
4021 bus_dmamap_unload(txr->txtag,
4023 m_freem(buf->m_head);
4033 txr->watchdog_time = ticks;
4035 /* Try the next packet */
4039 /* reset with a wrap */
4040 if (__predict_false(!work)) {
4041 work -= txr->num_desc;
4042 buf = txr->tx_buffers;
4046 } while (__predict_true(--limit));
4048 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4049 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4051 work += txr->num_desc;
4052 txr->next_to_clean = work;
4055 ** Watchdog calculation, we know there's
4056 ** work outstanding or the first return
4057 ** would have been taken, so none processed
4058 ** for too long indicates a hang.
4060 if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
4061 txr->queue_status |= IGB_QUEUE_HUNG;
4063 if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
4064 txr->queue_status &= ~IGB_QUEUE_DEPLETED;
4066 if (txr->tx_avail == txr->num_desc) {
4067 txr->queue_status = IGB_QUEUE_IDLE;
4074 /*********************************************************************
4076 * Refresh mbuf buffers for RX descriptor rings
4077 * - now keeps its own state so discards due to resource
4078 * exhaustion are unnecessary, if an mbuf cannot be obtained
4079 * it just returns, keeping its placeholder, thus it can simply
4080 * be recalled to try again.
4082 **********************************************************************/
4084 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
4086 struct adapter *adapter = rxr->adapter;
4087 bus_dma_segment_t hseg[1];
4088 bus_dma_segment_t pseg[1];
4089 struct igb_rx_buf *rxbuf;
4090 struct mbuf *mh, *mp;
4091 int i, j, nsegs, error;
4092 bool refreshed = FALSE;
4094 i = j = rxr->next_to_refresh;
4096 ** Get one descriptor beyond
4097 ** our work mark to control
4100 if (++j == adapter->num_rx_desc)
4103 while (j != limit) {
4104 rxbuf = &rxr->rx_buffers[i];
4105 /* No hdr mbuf used with header split off */
4106 if (rxr->hdr_split == FALSE)
4108 if (rxbuf->m_head == NULL) {
4109 mh = m_gethdr(M_NOWAIT, MT_DATA);
4115 mh->m_pkthdr.len = mh->m_len = MHLEN;
4117 mh->m_flags |= M_PKTHDR;
4118 /* Get the memory mapping */
4119 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4120 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
4122 printf("Refresh mbufs: hdr dmamap load"
4123 " failure - %d\n", error);
4125 rxbuf->m_head = NULL;
4129 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4130 BUS_DMASYNC_PREREAD);
4131 rxr->rx_base[i].read.hdr_addr =
4132 htole64(hseg[0].ds_addr);
4134 if (rxbuf->m_pack == NULL) {
4135 mp = m_getjcl(M_NOWAIT, MT_DATA,
4136 M_PKTHDR, adapter->rx_mbuf_sz);
4142 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4143 /* Get the memory mapping */
4144 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4145 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
4147 printf("Refresh mbufs: payload dmamap load"
4148 " failure - %d\n", error);
4150 rxbuf->m_pack = NULL;
4154 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4155 BUS_DMASYNC_PREREAD);
4156 rxr->rx_base[i].read.pkt_addr =
4157 htole64(pseg[0].ds_addr);
4158 refreshed = TRUE; /* I feel wefreshed :) */
4160 i = j; /* our next is precalculated */
4161 rxr->next_to_refresh = i;
4162 if (++j == adapter->num_rx_desc)
4166 if (refreshed) /* update tail */
4167 E1000_WRITE_REG(&adapter->hw,
4168 E1000_RDT(rxr->me), rxr->next_to_refresh);
4173 /*********************************************************************
4175 * Allocate memory for rx_buffer structures. Since we use one
4176 * rx_buffer per received packet, the maximum number of rx_buffer's
4177 * that we'll need is equal to the number of receive descriptors
4178 * that we've allocated.
4180 **********************************************************************/
4182 igb_allocate_receive_buffers(struct rx_ring *rxr)
4184 struct adapter *adapter = rxr->adapter;
4185 device_t dev = adapter->dev;
4186 struct igb_rx_buf *rxbuf;
4187 int i, bsize, error;
4189 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
4190 if (!(rxr->rx_buffers =
4191 (struct igb_rx_buf *) malloc(bsize,
4192 M_DEVBUF, M_NOWAIT | M_ZERO))) {
4193 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4198 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4199 1, 0, /* alignment, bounds */
4200 BUS_SPACE_MAXADDR, /* lowaddr */
4201 BUS_SPACE_MAXADDR, /* highaddr */
4202 NULL, NULL, /* filter, filterarg */
4203 MSIZE, /* maxsize */
4205 MSIZE, /* maxsegsize */
4207 NULL, /* lockfunc */
4208 NULL, /* lockfuncarg */
4210 device_printf(dev, "Unable to create RX DMA tag\n");
4214 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4215 1, 0, /* alignment, bounds */
4216 BUS_SPACE_MAXADDR, /* lowaddr */
4217 BUS_SPACE_MAXADDR, /* highaddr */
4218 NULL, NULL, /* filter, filterarg */
4219 MJUM9BYTES, /* maxsize */
4221 MJUM9BYTES, /* maxsegsize */
4223 NULL, /* lockfunc */
4224 NULL, /* lockfuncarg */
4226 device_printf(dev, "Unable to create RX payload DMA tag\n");
4230 for (i = 0; i < adapter->num_rx_desc; i++) {
4231 rxbuf = &rxr->rx_buffers[i];
4232 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
4235 "Unable to create RX head DMA maps\n");
4238 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
4241 "Unable to create RX packet DMA maps\n");
4249 /* Frees all, but can handle partial completion */
4250 igb_free_receive_structures(adapter);
4256 igb_free_receive_ring(struct rx_ring *rxr)
4258 struct adapter *adapter = rxr->adapter;
4259 struct igb_rx_buf *rxbuf;
4262 for (int i = 0; i < adapter->num_rx_desc; i++) {
4263 rxbuf = &rxr->rx_buffers[i];
4264 if (rxbuf->m_head != NULL) {
4265 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4266 BUS_DMASYNC_POSTREAD);
4267 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4268 rxbuf->m_head->m_flags |= M_PKTHDR;
4269 m_freem(rxbuf->m_head);
4271 if (rxbuf->m_pack != NULL) {
4272 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4273 BUS_DMASYNC_POSTREAD);
4274 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4275 rxbuf->m_pack->m_flags |= M_PKTHDR;
4276 m_freem(rxbuf->m_pack);
4278 rxbuf->m_head = NULL;
4279 rxbuf->m_pack = NULL;
4284 /*********************************************************************
4286 * Initialize a receive ring and its buffers.
4288 **********************************************************************/
4290 igb_setup_receive_ring(struct rx_ring *rxr)
4292 struct adapter *adapter;
4295 struct igb_rx_buf *rxbuf;
4296 bus_dma_segment_t pseg[1], hseg[1];
4297 struct lro_ctrl *lro = &rxr->lro;
4298 int rsize, nsegs, error = 0;
4300 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4301 struct netmap_slot *slot;
4302 #endif /* DEV_NETMAP */
4304 adapter = rxr->adapter;
4308 /* Clear the ring contents */
4311 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4312 #endif /* DEV_NETMAP */
4313 rsize = roundup2(adapter->num_rx_desc *
4314 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
4315 bzero((void *)rxr->rx_base, rsize);
4318 ** Free current RX buffer structures and their mbufs
4320 igb_free_receive_ring(rxr);
4322 /* Configure for header split? */
4323 if (igb_header_split)
4324 rxr->hdr_split = TRUE;
4326 /* Now replenish the ring mbufs */
4327 for (int j = 0; j < adapter->num_rx_desc; ++j) {
4328 struct mbuf *mh, *mp;
4330 rxbuf = &rxr->rx_buffers[j];
4333 /* slot sj is mapped to the i-th NIC-ring entry */
4334 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4338 addr = PNMB(slot + sj, &paddr);
4339 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4340 /* Update descriptor */
4341 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4344 #endif /* DEV_NETMAP */
4345 if (rxr->hdr_split == FALSE)
4348 /* First the header */
4349 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4350 if (rxbuf->m_head == NULL) {
4354 m_adj(rxbuf->m_head, ETHER_ALIGN);
4356 mh->m_len = mh->m_pkthdr.len = MHLEN;
4357 mh->m_flags |= M_PKTHDR;
4358 /* Get the memory mapping */
4359 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4360 rxbuf->hmap, rxbuf->m_head, hseg,
4361 &nsegs, BUS_DMA_NOWAIT);
4362 if (error != 0) /* Nothing elegant to do here */
4364 bus_dmamap_sync(rxr->htag,
4365 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4366 /* Update descriptor */
4367 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4370 /* Now the payload cluster */
4371 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4372 M_PKTHDR, adapter->rx_mbuf_sz);
4373 if (rxbuf->m_pack == NULL) {
4378 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4379 /* Get the memory mapping */
4380 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4381 rxbuf->pmap, mp, pseg,
4382 &nsegs, BUS_DMA_NOWAIT);
4385 bus_dmamap_sync(rxr->ptag,
4386 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4387 /* Update descriptor */
4388 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4391 /* Setup our descriptor indices */
4392 rxr->next_to_check = 0;
4393 rxr->next_to_refresh = adapter->num_rx_desc - 1;
4394 rxr->lro_enabled = FALSE;
4395 rxr->rx_split_packets = 0;
4400 rxr->discard = FALSE;
4402 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4403 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4406 ** Now set up the LRO interface, we
4407 ** also only do head split when LRO
4408 ** is enabled, since so often they
4409 ** are undesireable in similar setups.
4411 if (ifp->if_capenable & IFCAP_LRO) {
4412 error = tcp_lro_init(lro);
4414 device_printf(dev, "LRO Initialization failed!\n");
4417 INIT_DEBUGOUT("RX LRO Initialized\n");
4418 rxr->lro_enabled = TRUE;
4419 lro->ifp = adapter->ifp;
4426 igb_free_receive_ring(rxr);
4432 /*********************************************************************
4434 * Initialize all receive rings.
4436 **********************************************************************/
4438 igb_setup_receive_structures(struct adapter *adapter)
4440 struct rx_ring *rxr = adapter->rx_rings;
4443 for (i = 0; i < adapter->num_queues; i++, rxr++)
4444 if (igb_setup_receive_ring(rxr))
4450 * Free RX buffers allocated so far, we will only handle
4451 * the rings that completed, the failing case will have
4452 * cleaned up for itself. 'i' is the endpoint.
4454 for (int j = 0; j < i; ++j) {
4455 rxr = &adapter->rx_rings[j];
4457 igb_free_receive_ring(rxr);
4464 /*********************************************************************
4466 * Enable receive unit.
4468 **********************************************************************/
4470 igb_initialize_receive_units(struct adapter *adapter)
4472 struct rx_ring *rxr = adapter->rx_rings;
4473 struct ifnet *ifp = adapter->ifp;
4474 struct e1000_hw *hw = &adapter->hw;
4475 u32 rctl, rxcsum, psize, srrctl = 0;
4477 INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
4480 * Make sure receives are disabled while setting
4481 * up the descriptor ring
4483 rctl = E1000_READ_REG(hw, E1000_RCTL);
4484 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4487 ** Set up for header split
4489 if (igb_header_split) {
4490 /* Use a standard mbuf for the header */
4491 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4492 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4494 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4497 ** Set up for jumbo frames
4499 if (ifp->if_mtu > ETHERMTU) {
4500 rctl |= E1000_RCTL_LPE;
4501 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
4502 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4503 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4504 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
4505 srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4506 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4508 /* Set maximum packet len */
4509 psize = adapter->max_frame_size;
4510 /* are we on a vlan? */
4511 if (adapter->ifp->if_vlantrunk != NULL)
4512 psize += VLAN_TAG_SIZE;
4513 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
4515 rctl &= ~E1000_RCTL_LPE;
4516 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4517 rctl |= E1000_RCTL_SZ_2048;
4520 /* Setup the Base and Length of the Rx Descriptor Rings */
4521 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4522 u64 bus_addr = rxr->rxdma.dma_paddr;
4525 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4526 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4527 E1000_WRITE_REG(hw, E1000_RDBAH(i),
4528 (uint32_t)(bus_addr >> 32));
4529 E1000_WRITE_REG(hw, E1000_RDBAL(i),
4530 (uint32_t)bus_addr);
4531 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
4532 /* Enable this Queue */
4533 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
4534 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4535 rxdctl &= 0xFFF00000;
4536 rxdctl |= IGB_RX_PTHRESH;
4537 rxdctl |= IGB_RX_HTHRESH << 8;
4538 rxdctl |= IGB_RX_WTHRESH << 16;
4539 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
4543 ** Setup for RX MultiQueue
4545 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4546 if (adapter->num_queues >1) {
4547 u32 random[10], mrqc, shift = 0;
4553 arc4rand(&random, sizeof(random), 0);
4554 if (adapter->hw.mac.type == e1000_82575)
4556 /* Warning FM follows */
4557 for (int i = 0; i < 128; i++) {
4559 (i % adapter->num_queues) << shift;
4562 E1000_RETA(i >> 2), reta.dword);
4564 /* Now fill in hash table */
4565 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
4566 for (int i = 0; i < 10; i++)
4567 E1000_WRITE_REG_ARRAY(hw,
4568 E1000_RSSRK(0), i, random[i]);
4570 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
4571 E1000_MRQC_RSS_FIELD_IPV4_TCP);
4572 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
4573 E1000_MRQC_RSS_FIELD_IPV6_TCP);
4574 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
4575 E1000_MRQC_RSS_FIELD_IPV6_UDP);
4576 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
4577 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
4579 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
4582 ** NOTE: Receive Full-Packet Checksum Offload
4583 ** is mutually exclusive with Multiqueue. However
4584 ** this is not the same as TCP/IP checksums which
4587 rxcsum |= E1000_RXCSUM_PCSD;
4588 #if __FreeBSD_version >= 800000
4589 /* For SCTP Offload */
4590 if ((hw->mac.type == e1000_82576)
4591 && (ifp->if_capenable & IFCAP_RXCSUM))
4592 rxcsum |= E1000_RXCSUM_CRCOFL;
4596 if (ifp->if_capenable & IFCAP_RXCSUM) {
4597 rxcsum |= E1000_RXCSUM_IPPCSE;
4598 #if __FreeBSD_version >= 800000
4599 if (adapter->hw.mac.type == e1000_82576)
4600 rxcsum |= E1000_RXCSUM_CRCOFL;
4603 rxcsum &= ~E1000_RXCSUM_TUOFL;
4605 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4607 /* Setup the Receive Control Register */
4608 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4609 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4610 E1000_RCTL_RDMTS_HALF |
4611 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4612 /* Strip CRC bytes. */
4613 rctl |= E1000_RCTL_SECRC;
4614 /* Make sure VLAN Filters are off */
4615 rctl &= ~E1000_RCTL_VFE;
4616 /* Don't store bad packets */
4617 rctl &= ~E1000_RCTL_SBP;
4619 /* Enable Receives */
4620 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4623 * Setup the HW Rx Head and Tail Descriptor Pointers
4624 * - needs to be after enable
4626 for (int i = 0; i < adapter->num_queues; i++) {
4627 rxr = &adapter->rx_rings[i];
4628 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
4631 * an init() while a netmap client is active must
4632 * preserve the rx buffers passed to userspace.
4633 * In this driver it means we adjust RDT to
4634 * something different from next_to_refresh
4635 * (which is not used in netmap mode).
4637 if (ifp->if_capenable & IFCAP_NETMAP) {
4638 struct netmap_adapter *na = NA(adapter->ifp);
4639 struct netmap_kring *kring = &na->rx_rings[i];
4640 int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
4642 if (t >= adapter->num_rx_desc)
4643 t -= adapter->num_rx_desc;
4645 t += adapter->num_rx_desc;
4646 E1000_WRITE_REG(hw, E1000_RDT(i), t);
4648 #endif /* DEV_NETMAP */
4649 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
4654 /*********************************************************************
4656 * Free receive rings.
4658 **********************************************************************/
4660 igb_free_receive_structures(struct adapter *adapter)
4662 struct rx_ring *rxr = adapter->rx_rings;
4664 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4665 struct lro_ctrl *lro = &rxr->lro;
4666 igb_free_receive_buffers(rxr);
4668 igb_dma_free(adapter, &rxr->rxdma);
4671 free(adapter->rx_rings, M_DEVBUF);
4674 /*********************************************************************
4676 * Free receive ring data structures.
4678 **********************************************************************/
4680 igb_free_receive_buffers(struct rx_ring *rxr)
4682 struct adapter *adapter = rxr->adapter;
4683 struct igb_rx_buf *rxbuf;
4686 INIT_DEBUGOUT("free_receive_structures: begin");
4688 /* Cleanup any existing buffers */
4689 if (rxr->rx_buffers != NULL) {
4690 for (i = 0; i < adapter->num_rx_desc; i++) {
4691 rxbuf = &rxr->rx_buffers[i];
4692 if (rxbuf->m_head != NULL) {
4693 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4694 BUS_DMASYNC_POSTREAD);
4695 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4696 rxbuf->m_head->m_flags |= M_PKTHDR;
4697 m_freem(rxbuf->m_head);
4699 if (rxbuf->m_pack != NULL) {
4700 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4701 BUS_DMASYNC_POSTREAD);
4702 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4703 rxbuf->m_pack->m_flags |= M_PKTHDR;
4704 m_freem(rxbuf->m_pack);
4706 rxbuf->m_head = NULL;
4707 rxbuf->m_pack = NULL;
4708 if (rxbuf->hmap != NULL) {
4709 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4712 if (rxbuf->pmap != NULL) {
4713 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4717 if (rxr->rx_buffers != NULL) {
4718 free(rxr->rx_buffers, M_DEVBUF);
4719 rxr->rx_buffers = NULL;
4723 if (rxr->htag != NULL) {
4724 bus_dma_tag_destroy(rxr->htag);
4727 if (rxr->ptag != NULL) {
4728 bus_dma_tag_destroy(rxr->ptag);
4733 static __inline void
4734 igb_rx_discard(struct rx_ring *rxr, int i)
4736 struct igb_rx_buf *rbuf;
4738 rbuf = &rxr->rx_buffers[i];
4740 /* Partially received? Free the chain */
4741 if (rxr->fmp != NULL) {
4742 rxr->fmp->m_flags |= M_PKTHDR;
4749 ** With advanced descriptors the writeback
4750 ** clobbers the buffer addrs, so its easier
4751 ** to just free the existing mbufs and take
4752 ** the normal refresh path to get new buffers
4756 m_free(rbuf->m_head);
4757 rbuf->m_head = NULL;
4758 bus_dmamap_unload(rxr->htag, rbuf->hmap);
4762 m_free(rbuf->m_pack);
4763 rbuf->m_pack = NULL;
4764 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
4770 static __inline void
4771 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4775 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4776 * should be computed by hardware. Also it should not have VLAN tag in
4779 if (rxr->lro_enabled &&
4780 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4781 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4782 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4783 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4784 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4785 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4787 * Send to the stack if:
4788 ** - LRO not enabled, or
4789 ** - no LRO resources, or
4790 ** - lro enqueue fails
4792 if (rxr->lro.lro_cnt != 0)
4793 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4797 (*ifp->if_input)(ifp, m);
4801 /*********************************************************************
4803 * This routine executes in interrupt context. It replenishes
4804 * the mbufs in the descriptor and sends data which has been
4805 * dma'ed into host memory to upper layer.
4807 * We loop at most count times if count is > 0, or until done if
4810 * Return TRUE if more to clean, FALSE otherwise
4811 *********************************************************************/
4813 igb_rxeof(struct igb_queue *que, int count, int *done)
4815 struct adapter *adapter = que->adapter;
4816 struct rx_ring *rxr = que->rxr;
4817 struct ifnet *ifp = adapter->ifp;
4818 struct lro_ctrl *lro = &rxr->lro;
4819 struct lro_entry *queued;
4820 int i, processed = 0, rxdone = 0;
4821 u32 ptype, staterr = 0;
4822 union e1000_adv_rx_desc *cur;
4825 /* Sync the ring. */
4826 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4827 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4830 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4834 #endif /* DEV_NETMAP */
4836 /* Main clean loop */
4837 for (i = rxr->next_to_check; count != 0;) {
4838 struct mbuf *sendmp, *mh, *mp;
4839 struct igb_rx_buf *rxbuf;
4840 u16 hlen, plen, hdr, vtag;
4843 cur = &rxr->rx_base[i];
4844 staterr = le32toh(cur->wb.upper.status_error);
4845 if ((staterr & E1000_RXD_STAT_DD) == 0)
4847 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4850 sendmp = mh = mp = NULL;
4851 cur->wb.upper.status_error = 0;
4852 rxbuf = &rxr->rx_buffers[i];
4853 plen = le16toh(cur->wb.upper.length);
4854 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
4855 if (((adapter->hw.mac.type == e1000_i350) ||
4856 (adapter->hw.mac.type == e1000_i354)) &&
4857 (staterr & E1000_RXDEXT_STATERR_LB))
4858 vtag = be16toh(cur->wb.upper.vlan);
4860 vtag = le16toh(cur->wb.upper.vlan);
4861 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4862 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
4864 /* Make sure all segments of a bad packet are discarded */
4865 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) ||
4867 adapter->dropped_pkts++;
4868 ++rxr->rx_discarded;
4869 if (!eop) /* Catch subsequent segs */
4870 rxr->discard = TRUE;
4872 rxr->discard = FALSE;
4873 igb_rx_discard(rxr, i);
4878 ** The way the hardware is configured to
4879 ** split, it will ONLY use the header buffer
4880 ** when header split is enabled, otherwise we
4881 ** get normal behavior, ie, both header and
4882 ** payload are DMA'd into the payload buffer.
4884 ** The fmp test is to catch the case where a
4885 ** packet spans multiple descriptors, in that
4886 ** case only the first header is valid.
4888 if (rxr->hdr_split && rxr->fmp == NULL) {
4889 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4890 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4891 E1000_RXDADV_HDRBUFLEN_SHIFT;
4892 if (hlen > IGB_HDR_BUF)
4894 mh = rxr->rx_buffers[i].m_head;
4896 /* clear buf pointer for refresh */
4897 rxbuf->m_head = NULL;
4899 ** Get the payload length, this
4900 ** could be zero if its a small
4904 mp = rxr->rx_buffers[i].m_pack;
4907 /* clear buf pointer */
4908 rxbuf->m_pack = NULL;
4909 rxr->rx_split_packets++;
4913 ** Either no header split, or a
4914 ** secondary piece of a fragmented
4917 mh = rxr->rx_buffers[i].m_pack;
4919 /* clear buf info for refresh */
4920 rxbuf->m_pack = NULL;
4922 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4924 ++processed; /* So we know when to refresh */
4926 /* Initial frame - setup */
4927 if (rxr->fmp == NULL) {
4928 mh->m_pkthdr.len = mh->m_len;
4929 /* Save the head of the chain */
4933 /* Add payload if split */
4934 mh->m_pkthdr.len += mp->m_len;
4935 rxr->lmp = mh->m_next;
4938 /* Chain mbuf's together */
4939 rxr->lmp->m_next = mh;
4940 rxr->lmp = rxr->lmp->m_next;
4941 rxr->fmp->m_pkthdr.len += mh->m_len;
4945 rxr->fmp->m_pkthdr.rcvif = ifp;
4948 /* capture data for AIM */
4950 rxr->bytes += rxr->fmp->m_pkthdr.len;
4951 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4953 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4954 igb_rx_checksum(staterr, rxr->fmp, ptype);
4956 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4957 (staterr & E1000_RXD_STAT_VP) != 0) {
4958 rxr->fmp->m_pkthdr.ether_vtag = vtag;
4959 rxr->fmp->m_flags |= M_VLANTAG;
4961 #ifndef IGB_LEGACY_TX
4962 rxr->fmp->m_pkthdr.flowid = que->msix;
4963 rxr->fmp->m_flags |= M_FLOWID;
4966 /* Make sure to set M_PKTHDR. */
4967 sendmp->m_flags |= M_PKTHDR;
4973 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4974 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4976 /* Advance our pointers to the next descriptor. */
4977 if (++i == adapter->num_rx_desc)
4980 ** Send to the stack or LRO
4982 if (sendmp != NULL) {
4983 rxr->next_to_check = i;
4984 igb_rx_input(rxr, ifp, sendmp, ptype);
4985 i = rxr->next_to_check;
4989 /* Every 8 descriptors we go to refresh mbufs */
4990 if (processed == 8) {
4991 igb_refresh_mbufs(rxr, i);
4996 /* Catch any remainders */
4997 if (igb_rx_unrefreshed(rxr))
4998 igb_refresh_mbufs(rxr, i);
5000 rxr->next_to_check = i;
5003 * Flush any outstanding LRO work
5005 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
5006 SLIST_REMOVE_HEAD(&lro->lro_active, next);
5007 tcp_lro_flush(lro, queued);
5014 return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
5017 /*********************************************************************
5019 * Verify that the hardware indicated that the checksum is valid.
5020 * Inform the stack about the status of checksum so that stack
5021 * doesn't spend time verifying the checksum.
5023 *********************************************************************/
5025 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
5027 u16 status = (u16)staterr;
5028 u8 errors = (u8) (staterr >> 24);
5031 /* Ignore Checksum bit is set */
5032 if (status & E1000_RXD_STAT_IXSM) {
5033 mp->m_pkthdr.csum_flags = 0;
5037 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
5038 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
5042 if (status & E1000_RXD_STAT_IPCS) {
5044 if (!(errors & E1000_RXD_ERR_IPE)) {
5045 /* IP Checksum Good */
5046 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
5047 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
5049 mp->m_pkthdr.csum_flags = 0;
5052 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
5053 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5054 #if __FreeBSD_version >= 800000
5055 if (sctp) /* reassign */
5056 type = CSUM_SCTP_VALID;
5059 if (!(errors & E1000_RXD_ERR_TCPE)) {
5060 mp->m_pkthdr.csum_flags |= type;
5062 mp->m_pkthdr.csum_data = htons(0xffff);
5069 * This routine is run via an vlan
5073 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5075 struct adapter *adapter = ifp->if_softc;
5078 if (ifp->if_softc != arg) /* Not our event */
5081 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5084 IGB_CORE_LOCK(adapter);
5085 index = (vtag >> 5) & 0x7F;
5087 adapter->shadow_vfta[index] |= (1 << bit);
5088 ++adapter->num_vlans;
5089 /* Change hw filter setting */
5090 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5091 igb_setup_vlan_hw_support(adapter);
5092 IGB_CORE_UNLOCK(adapter);
5096 * This routine is run via an vlan
5100 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5102 struct adapter *adapter = ifp->if_softc;
5105 if (ifp->if_softc != arg)
5108 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5111 IGB_CORE_LOCK(adapter);
5112 index = (vtag >> 5) & 0x7F;
5114 adapter->shadow_vfta[index] &= ~(1 << bit);
5115 --adapter->num_vlans;
5116 /* Change hw filter setting */
5117 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5118 igb_setup_vlan_hw_support(adapter);
5119 IGB_CORE_UNLOCK(adapter);
5123 igb_setup_vlan_hw_support(struct adapter *adapter)
5125 struct e1000_hw *hw = &adapter->hw;
5126 struct ifnet *ifp = adapter->ifp;
5129 if (adapter->vf_ifp) {
5130 e1000_rlpml_set_vf(hw,
5131 adapter->max_frame_size + VLAN_TAG_SIZE);
5135 reg = E1000_READ_REG(hw, E1000_CTRL);
5136 reg |= E1000_CTRL_VME;
5137 E1000_WRITE_REG(hw, E1000_CTRL, reg);
5139 /* Enable the Filter Table */
5140 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
5141 reg = E1000_READ_REG(hw, E1000_RCTL);
5142 reg &= ~E1000_RCTL_CFIEN;
5143 reg |= E1000_RCTL_VFE;
5144 E1000_WRITE_REG(hw, E1000_RCTL, reg);
5147 /* Update the frame size */
5148 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
5149 adapter->max_frame_size + VLAN_TAG_SIZE);
5151 /* Don't bother with table if no vlans */
5152 if ((adapter->num_vlans == 0) ||
5153 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
5156 ** A soft reset zero's out the VFTA, so
5157 ** we need to repopulate it now.
5159 for (int i = 0; i < IGB_VFTA_SIZE; i++)
5160 if (adapter->shadow_vfta[i] != 0) {
5161 if (adapter->vf_ifp)
5162 e1000_vfta_set_vf(hw,
5163 adapter->shadow_vfta[i], TRUE);
5165 e1000_write_vfta(hw,
5166 i, adapter->shadow_vfta[i]);
5171 igb_enable_intr(struct adapter *adapter)
5173 /* With RSS set up what to auto clear */
5174 if (adapter->msix_mem) {
5175 u32 mask = (adapter->que_mask | adapter->link_mask);
5176 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
5177 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
5178 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
5179 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5182 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5185 E1000_WRITE_FLUSH(&adapter->hw);
5191 igb_disable_intr(struct adapter *adapter)
5193 if (adapter->msix_mem) {
5194 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
5195 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
5197 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
5198 E1000_WRITE_FLUSH(&adapter->hw);
5203 * Bit of a misnomer, what this really means is
5204 * to enable OS management of the system... aka
5205 * to disable special hardware management features
5208 igb_init_manageability(struct adapter *adapter)
5210 if (adapter->has_manage) {
5211 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
5212 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5214 /* disable hardware interception of ARP */
5215 manc &= ~(E1000_MANC_ARP_EN);
5217 /* enable receiving management packets to the host */
5218 manc |= E1000_MANC_EN_MNG2HOST;
5219 manc2h |= 1 << 5; /* Mng Port 623 */
5220 manc2h |= 1 << 6; /* Mng Port 664 */
5221 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
5222 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5227 * Give control back to hardware management
5228 * controller if there is one.
5231 igb_release_manageability(struct adapter *adapter)
5233 if (adapter->has_manage) {
5234 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5236 /* re-enable hardware interception of ARP */
5237 manc |= E1000_MANC_ARP_EN;
5238 manc &= ~E1000_MANC_EN_MNG2HOST;
5240 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5245 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
5246 * For ASF and Pass Through versions of f/w this means that
5247 * the driver is loaded.
5251 igb_get_hw_control(struct adapter *adapter)
5255 if (adapter->vf_ifp)
5258 /* Let firmware know the driver has taken over */
5259 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5260 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5261 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
5265 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
5266 * For ASF and Pass Through versions of f/w this means that the
5267 * driver is no longer loaded.
5271 igb_release_hw_control(struct adapter *adapter)
5275 if (adapter->vf_ifp)
5278 /* Let firmware taken over control of h/w */
5279 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5280 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5281 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
5285 igb_is_valid_ether_addr(uint8_t *addr)
5287 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
5289 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
5298 * Enable PCI Wake On Lan capability
5301 igb_enable_wakeup(device_t dev)
5306 /* First find the capabilities pointer*/
5307 cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
5308 /* Read the PM Capabilities */
5309 id = pci_read_config(dev, cap, 1);
5310 if (id != PCIY_PMG) /* Something wrong */
5312 /* OK, we have the power capabilities, so
5313 now get the status register */
5314 cap += PCIR_POWER_STATUS;
5315 status = pci_read_config(dev, cap, 2);
5316 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5317 pci_write_config(dev, cap, status, 2);
5322 igb_led_func(void *arg, int onoff)
5324 struct adapter *adapter = arg;
5326 IGB_CORE_LOCK(adapter);
5328 e1000_setup_led(&adapter->hw);
5329 e1000_led_on(&adapter->hw);
5331 e1000_led_off(&adapter->hw);
5332 e1000_cleanup_led(&adapter->hw);
5334 IGB_CORE_UNLOCK(adapter);
5337 /**********************************************************************
5339 * Update the board statistics counters.
5341 **********************************************************************/
5343 igb_update_stats_counters(struct adapter *adapter)
5346 struct e1000_hw *hw = &adapter->hw;
5347 struct e1000_hw_stats *stats;
5350 ** The virtual function adapter has only a
5351 ** small controlled set of stats, do only
5352 ** those and return.
5354 if (adapter->vf_ifp) {
5355 igb_update_vf_stats_counters(adapter);
5359 stats = (struct e1000_hw_stats *)adapter->stats;
5361 if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5362 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
5364 E1000_READ_REG(hw,E1000_SYMERRS);
5365 stats->sec += E1000_READ_REG(hw, E1000_SEC);
5368 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5369 stats->mpc += E1000_READ_REG(hw, E1000_MPC);
5370 stats->scc += E1000_READ_REG(hw, E1000_SCC);
5371 stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
5373 stats->mcc += E1000_READ_REG(hw, E1000_MCC);
5374 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
5375 stats->colc += E1000_READ_REG(hw, E1000_COLC);
5376 stats->dc += E1000_READ_REG(hw, E1000_DC);
5377 stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
5378 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5379 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5381 ** For watchdog management we need to know if we have been
5382 ** paused during the last interval, so capture that here.
5384 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5385 stats->xoffrxc += adapter->pause_frames;
5386 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5387 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5388 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
5389 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
5390 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
5391 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
5392 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5393 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5394 stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
5395 stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
5396 stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
5397 stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
5399 /* For the 64-bit byte counters the low dword must be read first. */
5400 /* Both registers clear on the read of the high dword */
5402 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
5403 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
5404 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
5405 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
5407 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
5408 stats->ruc += E1000_READ_REG(hw, E1000_RUC);
5409 stats->rfc += E1000_READ_REG(hw, E1000_RFC);
5410 stats->roc += E1000_READ_REG(hw, E1000_ROC);
5411 stats->rjc += E1000_READ_REG(hw, E1000_RJC);
5413 stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5414 stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC);
5415 stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5417 stats->tor += E1000_READ_REG(hw, E1000_TORL) +
5418 ((u64)E1000_READ_REG(hw, E1000_TORH) << 32);
5419 stats->tot += E1000_READ_REG(hw, E1000_TOTL) +
5420 ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32);
5422 stats->tpr += E1000_READ_REG(hw, E1000_TPR);
5423 stats->tpt += E1000_READ_REG(hw, E1000_TPT);
5424 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5425 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5426 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5427 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5428 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5429 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5430 stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
5431 stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
5433 /* Interrupt Counts */
5435 stats->iac += E1000_READ_REG(hw, E1000_IAC);
5436 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5437 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5438 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5439 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5440 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5441 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5442 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5443 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5445 /* Host to Card Statistics */
5447 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
5448 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
5449 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
5450 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
5451 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
5452 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
5453 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
5454 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
5455 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
5456 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
5457 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
5458 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
5459 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
5460 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
5462 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5463 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5464 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5465 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
5466 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5467 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5470 ifp->if_collisions = stats->colc;
5473 ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc +
5474 stats->crcerrs + stats->algnerrc +
5475 stats->ruc + stats->roc + stats->mpc + stats->cexterr;
5478 ifp->if_oerrors = stats->ecol +
5479 stats->latecol + adapter->watchdog_events;
5481 /* Driver specific counters */
5482 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
5483 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
5484 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
5485 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
5486 adapter->packet_buf_alloc_tx =
5487 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
5488 adapter->packet_buf_alloc_rx =
5489 (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
5493 /**********************************************************************
5495 * Initialize the VF board statistics counters.
5497 **********************************************************************/
5499 igb_vf_init_stats(struct adapter *adapter)
5501 struct e1000_hw *hw = &adapter->hw;
5502 struct e1000_vf_stats *stats;
5504 stats = (struct e1000_vf_stats *)adapter->stats;
5507 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
5508 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
5509 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
5510 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
5511 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
5514 /**********************************************************************
5516 * Update the VF board statistics counters.
5518 **********************************************************************/
5520 igb_update_vf_stats_counters(struct adapter *adapter)
5522 struct e1000_hw *hw = &adapter->hw;
5523 struct e1000_vf_stats *stats;
5525 if (adapter->link_speed == 0)
5528 stats = (struct e1000_vf_stats *)adapter->stats;
5530 UPDATE_VF_REG(E1000_VFGPRC,
5531 stats->last_gprc, stats->gprc);
5532 UPDATE_VF_REG(E1000_VFGORC,
5533 stats->last_gorc, stats->gorc);
5534 UPDATE_VF_REG(E1000_VFGPTC,
5535 stats->last_gptc, stats->gptc);
5536 UPDATE_VF_REG(E1000_VFGOTC,
5537 stats->last_gotc, stats->gotc);
5538 UPDATE_VF_REG(E1000_VFMPRC,
5539 stats->last_mprc, stats->mprc);
5542 /* Export a single 32-bit register via a read-only sysctl. */
5544 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5546 struct adapter *adapter;
5549 adapter = oidp->oid_arg1;
5550 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5551 return (sysctl_handle_int(oidp, &val, 0, req));
5555 ** Tuneable interrupt rate handler
5558 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5560 struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1);
5562 u32 reg, usec, rate;
5564 reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
5565 usec = ((reg & 0x7FFC) >> 2);
5567 rate = 1000000 / usec;
5570 error = sysctl_handle_int(oidp, &rate, 0, req);
5571 if (error || !req->newptr)
5577 * Add sysctl variables, one per statistic, to the system.
5580 igb_add_hw_stats(struct adapter *adapter)
5582 device_t dev = adapter->dev;
5584 struct tx_ring *txr = adapter->tx_rings;
5585 struct rx_ring *rxr = adapter->rx_rings;
5587 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5588 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5589 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5590 struct e1000_hw_stats *stats = adapter->stats;
5592 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
5593 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
5595 #define QUEUE_NAME_LEN 32
5596 char namebuf[QUEUE_NAME_LEN];
5598 /* Driver Statistics */
5599 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5600 CTLFLAG_RD, &adapter->link_irq,
5601 "Link MSIX IRQ Handled");
5602 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5603 CTLFLAG_RD, &adapter->dropped_pkts,
5604 "Driver dropped packets");
5605 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5606 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5607 "Driver tx dma failure in xmit");
5608 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5609 CTLFLAG_RD, &adapter->rx_overruns,
5611 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5612 CTLFLAG_RD, &adapter->watchdog_events,
5613 "Watchdog timeouts");
5615 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
5616 CTLFLAG_RD, &adapter->device_control,
5617 "Device Control Register");
5618 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
5619 CTLFLAG_RD, &adapter->rx_control,
5620 "Receiver Control Register");
5621 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
5622 CTLFLAG_RD, &adapter->int_mask,
5624 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
5625 CTLFLAG_RD, &adapter->eint_mask,
5626 "Extended Interrupt Mask");
5627 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
5628 CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
5629 "Transmit Buffer Packet Allocation");
5630 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
5631 CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
5632 "Receive Buffer Packet Allocation");
5633 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5634 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5635 "Flow Control High Watermark");
5636 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5637 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5638 "Flow Control Low Watermark");
5640 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5641 struct lro_ctrl *lro = &rxr->lro;
5643 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5644 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5645 CTLFLAG_RD, NULL, "Queue Name");
5646 queue_list = SYSCTL_CHILDREN(queue_node);
5648 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5649 CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i],
5650 sizeof(&adapter->queues[i]),
5651 igb_sysctl_interrupt_rate_handler,
5652 "IU", "Interrupt Rate");
5654 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5655 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me),
5656 igb_sysctl_reg_handler, "IU",
5657 "Transmit Descriptor Head");
5658 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5659 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me),
5660 igb_sysctl_reg_handler, "IU",
5661 "Transmit Descriptor Tail");
5662 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5663 CTLFLAG_RD, &txr->no_desc_avail,
5664 "Queue No Descriptor Available");
5665 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5666 CTLFLAG_RD, &txr->total_packets,
5667 "Queue Packets Transmitted");
5669 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5670 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
5671 igb_sysctl_reg_handler, "IU",
5672 "Receive Descriptor Head");
5673 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5674 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
5675 igb_sysctl_reg_handler, "IU",
5676 "Receive Descriptor Tail");
5677 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5678 CTLFLAG_RD, &rxr->rx_packets,
5679 "Queue Packets Received");
5680 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5681 CTLFLAG_RD, &rxr->rx_bytes,
5682 "Queue Bytes Received");
5683 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
5684 CTLFLAG_RD, &lro->lro_queued, 0,
5686 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
5687 CTLFLAG_RD, &lro->lro_flushed, 0,
5691 /* MAC stats get their own sub node */
5693 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5694 CTLFLAG_RD, NULL, "MAC Statistics");
5695 stat_list = SYSCTL_CHILDREN(stat_node);
5698 ** VF adapter has a very limited set of stats
5699 ** since its not managing the metal, so to speak.
5701 if (adapter->vf_ifp) {
5702 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5703 CTLFLAG_RD, &stats->gprc,
5704 "Good Packets Received");
5705 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5706 CTLFLAG_RD, &stats->gptc,
5707 "Good Packets Transmitted");
5708 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5709 CTLFLAG_RD, &stats->gorc,
5710 "Good Octets Received");
5711 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5712 CTLFLAG_RD, &stats->gotc,
5713 "Good Octets Transmitted");
5714 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5715 CTLFLAG_RD, &stats->mprc,
5716 "Multicast Packets Received");
5720 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5721 CTLFLAG_RD, &stats->ecol,
5722 "Excessive collisions");
5723 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
5724 CTLFLAG_RD, &stats->scc,
5725 "Single collisions");
5726 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5727 CTLFLAG_RD, &stats->mcc,
5728 "Multiple collisions");
5729 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
5730 CTLFLAG_RD, &stats->latecol,
5732 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
5733 CTLFLAG_RD, &stats->colc,
5735 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5736 CTLFLAG_RD, &stats->symerrs,
5738 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5739 CTLFLAG_RD, &stats->sec,
5741 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
5742 CTLFLAG_RD, &stats->dc,
5744 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5745 CTLFLAG_RD, &stats->mpc,
5747 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors",
5748 CTLFLAG_RD, &stats->rlec,
5749 "Receive Length Errors");
5750 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5751 CTLFLAG_RD, &stats->rnbc,
5752 "Receive No Buffers");
5753 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5754 CTLFLAG_RD, &stats->ruc,
5755 "Receive Undersize");
5756 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5757 CTLFLAG_RD, &stats->rfc,
5758 "Fragmented Packets Received");
5759 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5760 CTLFLAG_RD, &stats->roc,
5761 "Oversized Packets Received");
5762 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5763 CTLFLAG_RD, &stats->rjc,
5765 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5766 CTLFLAG_RD, &stats->rxerrc,
5768 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5769 CTLFLAG_RD, &stats->crcerrs,
5771 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5772 CTLFLAG_RD, &stats->algnerrc,
5773 "Alignment Errors");
5774 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs",
5775 CTLFLAG_RD, &stats->tncrs,
5776 "Transmit with No CRS");
5777 /* On 82575 these are collision counts */
5778 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5779 CTLFLAG_RD, &stats->cexterr,
5780 "Collision/Carrier extension errors");
5781 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5782 CTLFLAG_RD, &stats->xonrxc,
5784 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5785 CTLFLAG_RD, &stats->xontxc,
5787 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5788 CTLFLAG_RD, &stats->xoffrxc,
5790 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5791 CTLFLAG_RD, &stats->xofftxc,
5792 "XOFF Transmitted");
5793 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd",
5794 CTLFLAG_RD, &stats->fcruc,
5795 "Unsupported Flow Control Received");
5796 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd",
5797 CTLFLAG_RD, &stats->mgprc,
5798 "Management Packets Received");
5799 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop",
5800 CTLFLAG_RD, &stats->mgpdc,
5801 "Management Packets Dropped");
5802 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd",
5803 CTLFLAG_RD, &stats->mgptc,
5804 "Management Packets Transmitted");
5805 /* Packet Reception Stats */
5806 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5807 CTLFLAG_RD, &stats->tpr,
5808 "Total Packets Received");
5809 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5810 CTLFLAG_RD, &stats->gprc,
5811 "Good Packets Received");
5812 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5813 CTLFLAG_RD, &stats->bprc,
5814 "Broadcast Packets Received");
5815 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5816 CTLFLAG_RD, &stats->mprc,
5817 "Multicast Packets Received");
5818 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5819 CTLFLAG_RD, &stats->prc64,
5820 "64 byte frames received");
5821 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5822 CTLFLAG_RD, &stats->prc127,
5823 "65-127 byte frames received");
5824 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5825 CTLFLAG_RD, &stats->prc255,
5826 "128-255 byte frames received");
5827 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5828 CTLFLAG_RD, &stats->prc511,
5829 "256-511 byte frames received");
5830 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5831 CTLFLAG_RD, &stats->prc1023,
5832 "512-1023 byte frames received");
5833 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5834 CTLFLAG_RD, &stats->prc1522,
5835 "1023-1522 byte frames received");
5836 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5837 CTLFLAG_RD, &stats->gorc,
5838 "Good Octets Received");
5839 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd",
5840 CTLFLAG_RD, &stats->tor,
5841 "Total Octets Received");
5843 /* Packet Transmission Stats */
5844 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5845 CTLFLAG_RD, &stats->gotc,
5846 "Good Octets Transmitted");
5847 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd",
5848 CTLFLAG_RD, &stats->tot,
5849 "Total Octets Transmitted");
5850 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5851 CTLFLAG_RD, &stats->tpt,
5852 "Total Packets Transmitted");
5853 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5854 CTLFLAG_RD, &stats->gptc,
5855 "Good Packets Transmitted");
5856 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5857 CTLFLAG_RD, &stats->bptc,
5858 "Broadcast Packets Transmitted");
5859 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5860 CTLFLAG_RD, &stats->mptc,
5861 "Multicast Packets Transmitted");
5862 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5863 CTLFLAG_RD, &stats->ptc64,
5864 "64 byte frames transmitted");
5865 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5866 CTLFLAG_RD, &stats->ptc127,
5867 "65-127 byte frames transmitted");
5868 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5869 CTLFLAG_RD, &stats->ptc255,
5870 "128-255 byte frames transmitted");
5871 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5872 CTLFLAG_RD, &stats->ptc511,
5873 "256-511 byte frames transmitted");
5874 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5875 CTLFLAG_RD, &stats->ptc1023,
5876 "512-1023 byte frames transmitted");
5877 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5878 CTLFLAG_RD, &stats->ptc1522,
5879 "1024-1522 byte frames transmitted");
5880 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5881 CTLFLAG_RD, &stats->tsctc,
5882 "TSO Contexts Transmitted");
5883 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5884 CTLFLAG_RD, &stats->tsctfc,
5885 "TSO Contexts Failed");
5888 /* Interrupt Stats */
5890 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5891 CTLFLAG_RD, NULL, "Interrupt Statistics");
5892 int_list = SYSCTL_CHILDREN(int_node);
5894 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
5895 CTLFLAG_RD, &stats->iac,
5896 "Interrupt Assertion Count");
5898 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5899 CTLFLAG_RD, &stats->icrxptc,
5900 "Interrupt Cause Rx Pkt Timer Expire Count");
5902 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5903 CTLFLAG_RD, &stats->icrxatc,
5904 "Interrupt Cause Rx Abs Timer Expire Count");
5906 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5907 CTLFLAG_RD, &stats->ictxptc,
5908 "Interrupt Cause Tx Pkt Timer Expire Count");
5910 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5911 CTLFLAG_RD, &stats->ictxatc,
5912 "Interrupt Cause Tx Abs Timer Expire Count");
5914 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5915 CTLFLAG_RD, &stats->ictxqec,
5916 "Interrupt Cause Tx Queue Empty Count");
5918 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5919 CTLFLAG_RD, &stats->ictxqmtc,
5920 "Interrupt Cause Tx Queue Min Thresh Count");
5922 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5923 CTLFLAG_RD, &stats->icrxdmtc,
5924 "Interrupt Cause Rx Desc Min Thresh Count");
5926 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5927 CTLFLAG_RD, &stats->icrxoc,
5928 "Interrupt Cause Receiver Overrun Count");
5930 /* Host to Card Stats */
5932 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
5934 "Host to Card Statistics");
5936 host_list = SYSCTL_CHILDREN(host_node);
5938 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
5939 CTLFLAG_RD, &stats->cbtmpc,
5940 "Circuit Breaker Tx Packet Count");
5942 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
5943 CTLFLAG_RD, &stats->htdpmc,
5944 "Host Transmit Discarded Packets");
5946 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
5947 CTLFLAG_RD, &stats->rpthc,
5948 "Rx Packets To Host");
5950 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
5951 CTLFLAG_RD, &stats->cbrmpc,
5952 "Circuit Breaker Rx Packet Count");
5954 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
5955 CTLFLAG_RD, &stats->cbrdpc,
5956 "Circuit Breaker Rx Dropped Count");
5958 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
5959 CTLFLAG_RD, &stats->hgptc,
5960 "Host Good Packets Tx Count");
5962 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
5963 CTLFLAG_RD, &stats->htcbdpc,
5964 "Host Tx Circuit Breaker Dropped Count");
5966 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
5967 CTLFLAG_RD, &stats->hgorc,
5968 "Host Good Octets Received Count");
5970 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
5971 CTLFLAG_RD, &stats->hgotc,
5972 "Host Good Octets Transmit Count");
5974 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
5975 CTLFLAG_RD, &stats->lenerrs,
5978 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
5979 CTLFLAG_RD, &stats->scvpc,
5980 "SerDes/SGMII Code Violation Pkt Count");
5982 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
5983 CTLFLAG_RD, &stats->hrmpc,
5984 "Header Redirection Missed Packet Count");
5988 /**********************************************************************
5990 * This routine provides a way to dump out the adapter eeprom,
5991 * often a useful debug/service tool. This only dumps the first
5992 * 32 words, stuff that matters is in that extent.
5994 **********************************************************************/
5996 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5998 struct adapter *adapter;
6003 error = sysctl_handle_int(oidp, &result, 0, req);
6005 if (error || !req->newptr)
6009 * This value will cause a hex dump of the
6010 * first 32 16-bit words of the EEPROM to
6014 adapter = (struct adapter *)arg1;
6015 igb_print_nvm_info(adapter);
6022 igb_print_nvm_info(struct adapter *adapter)
6027 /* Its a bit crude, but it gets the job done */
6028 printf("\nInterface EEPROM Dump:\n");
6029 printf("Offset\n0x0000 ");
6030 for (i = 0, j = 0; i < 32; i++, j++) {
6031 if (j == 8) { /* Make the offset block */
6033 printf("\n0x00%x0 ",row);
6035 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
6036 printf("%04x ", eeprom_data);
6042 igb_set_sysctl_value(struct adapter *adapter, const char *name,
6043 const char *description, int *limit, int value)
6046 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
6047 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
6048 OID_AUTO, name, CTLFLAG_RW, limit, value, description);
6052 ** Set flow control using sysctl:
6053 ** Flow control values:
6060 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
6063 static int input = 3; /* default is full */
6064 struct adapter *adapter = (struct adapter *) arg1;
6066 error = sysctl_handle_int(oidp, &input, 0, req);
6068 if ((error) || (req->newptr == NULL))
6072 case e1000_fc_rx_pause:
6073 case e1000_fc_tx_pause:
6076 adapter->hw.fc.requested_mode = input;
6077 adapter->fc = input;
6084 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
6085 e1000_force_mac_fc(&adapter->hw);
6090 ** Manage DMA Coalesce:
6093 ** Legal timer values are:
6094 ** 250,500,1000-10000 in thousands
6097 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
6099 struct adapter *adapter = (struct adapter *) arg1;
6102 error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
6104 if ((error) || (req->newptr == NULL))
6107 switch (adapter->dmac) {
6111 case 1: /* Just enable and use default */
6112 adapter->dmac = 1000;
6126 /* Legal values - allow */
6129 /* Do nothing, illegal value */
6133 /* Reinit the interface */
6139 ** Manage Energy Efficient Ethernet:
6141 ** 0/1 - enabled/disabled
6144 igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
6146 struct adapter *adapter = (struct adapter *) arg1;
6149 value = adapter->hw.dev_spec._82575.eee_disable;
6150 error = sysctl_handle_int(oidp, &value, 0, req);
6151 if (error || req->newptr == NULL)
6153 IGB_CORE_LOCK(adapter);
6154 adapter->hw.dev_spec._82575.eee_disable = (value != 0);
6155 igb_init_locked(adapter);
6156 IGB_CORE_UNLOCK(adapter);