1 /******************************************************************************
3 Copyright (c) 2001-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
39 #ifdef HAVE_KERNEL_OPTION_HEADERS
40 #include "opt_device_polling.h"
46 /*********************************************************************
48 *********************************************************************/
49 char igb_driver_version[] = "2.5.3-k";
52 /*********************************************************************
55 * Used by probe to select devices to load on
56 * Last field stores an index into e1000_strings
57 * Last entry must be all 0s
59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
60 *********************************************************************/
62 static igb_vendor_info_t igb_vendor_info_array[] =
64 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0},
65 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0},
66 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0},
67 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0},
68 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0},
69 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0},
70 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0},
71 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0},
72 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0},
73 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0},
74 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0},
75 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0},
76 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0},
77 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0},
78 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0},
79 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0},
80 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0},
81 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0},
82 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0},
83 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0},
84 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0},
85 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0},
86 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0},
87 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0},
88 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0},
89 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0},
90 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0},
91 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0},
92 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0},
93 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0},
94 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0},
95 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0},
96 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0},
97 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0},
98 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0},
99 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0},
100 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0},
101 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0},
102 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0},
103 /* required last entry */
107 /*********************************************************************
108 * Table of branding strings for all supported NICs.
109 *********************************************************************/
111 static char *igb_strings[] = {
112 "Intel(R) PRO/1000 Network Connection"
115 /*********************************************************************
116 * Function prototypes
117 *********************************************************************/
118 static int igb_probe(device_t);
119 static int igb_attach(device_t);
120 static int igb_detach(device_t);
121 static int igb_shutdown(device_t);
122 static int igb_suspend(device_t);
123 static int igb_resume(device_t);
124 #ifndef IGB_LEGACY_TX
125 static int igb_mq_start(struct ifnet *, struct mbuf *);
126 static int igb_mq_start_locked(struct ifnet *, struct tx_ring *);
127 static void igb_qflush(struct ifnet *);
128 static void igb_deferred_mq_start(void *, int);
130 static void igb_start(struct ifnet *);
131 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
133 static int igb_ioctl(struct ifnet *, u_long, caddr_t);
134 static void igb_init(void *);
135 static void igb_init_locked(struct adapter *);
136 static void igb_stop(void *);
137 static void igb_media_status(struct ifnet *, struct ifmediareq *);
138 static int igb_media_change(struct ifnet *);
139 static void igb_identify_hardware(struct adapter *);
140 static int igb_allocate_pci_resources(struct adapter *);
141 static int igb_allocate_msix(struct adapter *);
142 static int igb_allocate_legacy(struct adapter *);
143 static int igb_setup_msix(struct adapter *);
144 static void igb_free_pci_resources(struct adapter *);
145 static void igb_local_timer(void *);
146 static void igb_reset(struct adapter *);
147 static int igb_setup_interface(device_t, struct adapter *);
148 static int igb_allocate_queues(struct adapter *);
149 static void igb_configure_queues(struct adapter *);
151 static int igb_allocate_transmit_buffers(struct tx_ring *);
152 static void igb_setup_transmit_structures(struct adapter *);
153 static void igb_setup_transmit_ring(struct tx_ring *);
154 static void igb_initialize_transmit_units(struct adapter *);
155 static void igb_free_transmit_structures(struct adapter *);
156 static void igb_free_transmit_buffers(struct tx_ring *);
158 static int igb_allocate_receive_buffers(struct rx_ring *);
159 static int igb_setup_receive_structures(struct adapter *);
160 static int igb_setup_receive_ring(struct rx_ring *);
161 static void igb_initialize_receive_units(struct adapter *);
162 static void igb_free_receive_structures(struct adapter *);
163 static void igb_free_receive_buffers(struct rx_ring *);
164 static void igb_free_receive_ring(struct rx_ring *);
166 static void igb_enable_intr(struct adapter *);
167 static void igb_disable_intr(struct adapter *);
168 static void igb_update_stats_counters(struct adapter *);
169 static bool igb_txeof(struct tx_ring *);
171 static __inline void igb_rx_discard(struct rx_ring *, int);
172 static __inline void igb_rx_input(struct rx_ring *,
173 struct ifnet *, struct mbuf *, u32);
175 static bool igb_rxeof(struct igb_queue *, int, int *);
176 static void igb_rx_checksum(u32, struct mbuf *, u32);
177 static int igb_tx_ctx_setup(struct tx_ring *,
178 struct mbuf *, u32 *, u32 *);
179 static int igb_tso_setup(struct tx_ring *,
180 struct mbuf *, u32 *, u32 *);
181 static void igb_set_promisc(struct adapter *);
182 static void igb_disable_promisc(struct adapter *);
183 static void igb_set_multi(struct adapter *);
184 static void igb_update_link_status(struct adapter *);
185 static void igb_refresh_mbufs(struct rx_ring *, int);
187 static void igb_register_vlan(void *, struct ifnet *, u16);
188 static void igb_unregister_vlan(void *, struct ifnet *, u16);
189 static void igb_setup_vlan_hw_support(struct adapter *);
191 static int igb_xmit(struct tx_ring *, struct mbuf **);
192 static int igb_dma_malloc(struct adapter *, bus_size_t,
193 struct igb_dma_alloc *, int);
194 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
195 static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
196 static void igb_print_nvm_info(struct adapter *);
197 static int igb_is_valid_ether_addr(u8 *);
198 static void igb_add_hw_stats(struct adapter *);
200 static void igb_vf_init_stats(struct adapter *);
201 static void igb_update_vf_stats_counters(struct adapter *);
203 /* Management and WOL Support */
204 static void igb_init_manageability(struct adapter *);
205 static void igb_release_manageability(struct adapter *);
206 static void igb_get_hw_control(struct adapter *);
207 static void igb_release_hw_control(struct adapter *);
208 static void igb_enable_wakeup(device_t);
209 static void igb_led_func(void *, int);
211 static int igb_irq_fast(void *);
212 static void igb_msix_que(void *);
213 static void igb_msix_link(void *);
214 static void igb_handle_que(void *context, int pending);
215 static void igb_handle_link(void *context, int pending);
216 static void igb_handle_link_locked(struct adapter *);
218 static void igb_set_sysctl_value(struct adapter *, const char *,
219 const char *, int *, int);
220 static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
221 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
222 static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
224 #ifdef DEVICE_POLLING
225 static poll_handler_t igb_poll;
228 /*********************************************************************
229 * FreeBSD Device Interface Entry Points
230 *********************************************************************/
232 static device_method_t igb_methods[] = {
233 /* Device interface */
234 DEVMETHOD(device_probe, igb_probe),
235 DEVMETHOD(device_attach, igb_attach),
236 DEVMETHOD(device_detach, igb_detach),
237 DEVMETHOD(device_shutdown, igb_shutdown),
238 DEVMETHOD(device_suspend, igb_suspend),
239 DEVMETHOD(device_resume, igb_resume),
243 static driver_t igb_driver = {
244 "igb", igb_methods, sizeof(struct adapter),
247 static devclass_t igb_devclass;
248 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
249 MODULE_DEPEND(igb, pci, 1, 1, 1);
250 MODULE_DEPEND(igb, ether, 1, 1, 1);
252 /*********************************************************************
253 * Tunable default values.
254 *********************************************************************/
256 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
258 /* Descriptor defaults */
259 static int igb_rxd = IGB_DEFAULT_RXD;
260 static int igb_txd = IGB_DEFAULT_TXD;
261 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
262 TUNABLE_INT("hw.igb.txd", &igb_txd);
263 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
264 "Number of receive descriptors per queue");
265 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
266 "Number of transmit descriptors per queue");
269 ** AIM: Adaptive Interrupt Moderation
270 ** which means that the interrupt rate
271 ** is varied over time based on the
272 ** traffic for that interrupt vector
274 static int igb_enable_aim = TRUE;
275 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
276 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0,
277 "Enable adaptive interrupt moderation");
280 * MSIX should be the default for best performance,
281 * but this allows it to be forced off for testing.
283 static int igb_enable_msix = 1;
284 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
285 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
286 "Enable MSI-X interrupts");
289 ** Tuneable Interrupt rate
291 static int igb_max_interrupt_rate = 8000;
292 TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate);
293 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
294 &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
296 #ifndef IGB_LEGACY_TX
298 ** Tuneable number of buffers in the buf-ring (drbr_xxx)
300 static int igb_buf_ring_size = IGB_BR_SIZE;
301 TUNABLE_INT("hw.igb.buf_ring_size", &igb_buf_ring_size);
302 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
303 &igb_buf_ring_size, 0, "Size of the bufring");
307 ** Header split causes the packet header to
308 ** be dma'd to a seperate mbuf from the payload.
309 ** this can have memory alignment benefits. But
310 ** another plus is that small packets often fit
311 ** into the header and thus use no cluster. Its
312 ** a very workload dependent type feature.
314 static int igb_header_split = FALSE;
315 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
316 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
317 "Enable receive mbuf header split");
320 ** This will autoconfigure based on the
321 ** number of CPUs and max supported
322 ** MSIX messages if left at 0.
324 static int igb_num_queues = 0;
325 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
326 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
327 "Number of queues to configure, 0 indicates autoconfigure");
330 ** Global variable to store last used CPU when binding queues
331 ** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
332 ** queue is bound to a cpu.
334 static int igb_last_bind_cpu = -1;
336 /* How many packets rxeof tries to clean at a time */
337 static int igb_rx_process_limit = 100;
338 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
339 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
340 &igb_rx_process_limit, 0,
341 "Maximum number of received packets to process at a time, -1 means unlimited");
343 /* How many packets txeof tries to clean at a time */
344 static int igb_tx_process_limit = -1;
345 TUNABLE_INT("hw.igb.tx_process_limit", &igb_tx_process_limit);
346 SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
347 &igb_tx_process_limit, 0,
348 "Maximum number of sent packets to process at a time, -1 means unlimited");
350 #ifdef DEV_NETMAP /* see ixgbe.c for details */
351 #include <dev/netmap/if_igb_netmap.h>
352 #endif /* DEV_NETMAP */
353 /*********************************************************************
354 * Device identification routine
356 * igb_probe determines if the driver should be loaded on
357 * adapter based on PCI vendor/device id of the adapter.
359 * return BUS_PROBE_DEFAULT on success, positive on failure
360 *********************************************************************/
363 igb_probe(device_t dev)
365 char adapter_name[256];
366 uint16_t pci_vendor_id = 0;
367 uint16_t pci_device_id = 0;
368 uint16_t pci_subvendor_id = 0;
369 uint16_t pci_subdevice_id = 0;
370 igb_vendor_info_t *ent;
372 INIT_DEBUGOUT("igb_probe: begin");
374 pci_vendor_id = pci_get_vendor(dev);
375 if (pci_vendor_id != IGB_INTEL_VENDOR_ID)
378 pci_device_id = pci_get_device(dev);
379 pci_subvendor_id = pci_get_subvendor(dev);
380 pci_subdevice_id = pci_get_subdevice(dev);
382 ent = igb_vendor_info_array;
383 while (ent->vendor_id != 0) {
384 if ((pci_vendor_id == ent->vendor_id) &&
385 (pci_device_id == ent->device_id) &&
387 ((pci_subvendor_id == ent->subvendor_id) ||
388 (ent->subvendor_id == 0)) &&
390 ((pci_subdevice_id == ent->subdevice_id) ||
391 (ent->subdevice_id == 0))) {
392 sprintf(adapter_name, "%s, Version - %s",
393 igb_strings[ent->index],
395 device_set_desc_copy(dev, adapter_name);
396 return (BUS_PROBE_DEFAULT);
403 /*********************************************************************
404 * Device initialization routine
406 * The attach entry point is called when the driver is being loaded.
407 * This routine identifies the type of hardware, allocates all resources
408 * and initializes the hardware.
410 * return 0 on success, positive on failure
411 *********************************************************************/
414 igb_attach(device_t dev)
416 struct adapter *adapter;
420 INIT_DEBUGOUT("igb_attach: begin");
422 if (resource_disabled("igb", device_get_unit(dev))) {
423 device_printf(dev, "Disabled by device hint\n");
427 adapter = device_get_softc(dev);
428 adapter->dev = adapter->osdep.dev = dev;
429 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
432 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
433 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
434 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
435 igb_sysctl_nvm_info, "I", "NVM Information");
437 igb_set_sysctl_value(adapter, "enable_aim",
438 "Interrupt Moderation", &adapter->enable_aim,
441 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
442 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
443 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
444 adapter, 0, igb_set_flowcntl, "I", "Flow Control");
446 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
448 /* Determine hardware and mac info */
449 igb_identify_hardware(adapter);
451 /* Setup PCI resources */
452 if (igb_allocate_pci_resources(adapter)) {
453 device_printf(dev, "Allocation of PCI resources failed\n");
458 /* Do Shared Code initialization */
459 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
460 device_printf(dev, "Setup of Shared code failed\n");
465 e1000_get_bus_info(&adapter->hw);
467 /* Sysctls for limiting the amount of work done in the taskqueues */
468 igb_set_sysctl_value(adapter, "rx_processing_limit",
469 "max number of rx packets to process",
470 &adapter->rx_process_limit, igb_rx_process_limit);
472 igb_set_sysctl_value(adapter, "tx_processing_limit",
473 "max number of tx packets to process",
474 &adapter->tx_process_limit, igb_tx_process_limit);
477 * Validate number of transmit and receive descriptors. It
478 * must not exceed hardware maximum, and must be multiple
479 * of E1000_DBA_ALIGN.
481 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
482 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
483 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
484 IGB_DEFAULT_TXD, igb_txd);
485 adapter->num_tx_desc = IGB_DEFAULT_TXD;
487 adapter->num_tx_desc = igb_txd;
488 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
489 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
490 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
491 IGB_DEFAULT_RXD, igb_rxd);
492 adapter->num_rx_desc = IGB_DEFAULT_RXD;
494 adapter->num_rx_desc = igb_rxd;
496 adapter->hw.mac.autoneg = DO_AUTO_NEG;
497 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
498 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
501 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
502 adapter->hw.phy.mdix = AUTO_ALL_MODES;
503 adapter->hw.phy.disable_polarity_correction = FALSE;
504 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
508 * Set the frame limits assuming
509 * standard ethernet sized frames.
511 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
514 ** Allocate and Setup Queues
516 if (igb_allocate_queues(adapter)) {
521 /* Allocate the appropriate stats memory */
522 if (adapter->vf_ifp) {
524 (struct e1000_vf_stats *)malloc(sizeof \
525 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
526 igb_vf_init_stats(adapter);
529 (struct e1000_hw_stats *)malloc(sizeof \
530 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
531 if (adapter->stats == NULL) {
532 device_printf(dev, "Can not allocate stats memory\n");
537 /* Allocate multicast array memory. */
538 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
539 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
540 if (adapter->mta == NULL) {
541 device_printf(dev, "Can not allocate multicast setup array\n");
546 /* Some adapter-specific advanced features */
547 if (adapter->hw.mac.type >= e1000_i350) {
548 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
549 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
550 OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
551 adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
552 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
553 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
554 OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
555 adapter, 0, igb_sysctl_eee, "I",
556 "Disable Energy Efficient Ethernet");
557 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
558 if (adapter->hw.mac.type == e1000_i354)
559 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
561 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
566 ** Start from a known state, this is
567 ** important in reading the nvm and
570 e1000_reset_hw(&adapter->hw);
572 /* Make sure we have a good EEPROM before we read from it */
573 if (((adapter->hw.mac.type != e1000_i210) &&
574 (adapter->hw.mac.type != e1000_i211)) &&
575 (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
577 ** Some PCI-E parts fail the first check due to
578 ** the link being in sleep state, call it again,
579 ** if it fails a second time its a real issue.
581 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
583 "The EEPROM Checksum Is Not Valid\n");
590 ** Copy the permanent MAC address out of the EEPROM
592 if (e1000_read_mac_addr(&adapter->hw) < 0) {
593 device_printf(dev, "EEPROM read error while reading MAC"
598 /* Check its sanity */
599 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
600 device_printf(dev, "Invalid MAC address\n");
605 /* Setup OS specific network interface */
606 if (igb_setup_interface(dev, adapter) != 0)
609 /* Now get a good starting state */
612 /* Initialize statistics */
613 igb_update_stats_counters(adapter);
615 adapter->hw.mac.get_link_status = 1;
616 igb_update_link_status(adapter);
618 /* Indicate SOL/IDER usage */
619 if (e1000_check_reset_block(&adapter->hw))
621 "PHY reset is blocked due to SOL/IDER session.\n");
623 /* Determine if we have to control management hardware */
624 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
629 /* APME bit in EEPROM is mapped to WUC.APME */
630 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
632 adapter->wol = E1000_WUFC_MAG;
634 /* Register for VLAN events */
635 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
636 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
637 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
638 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
640 igb_add_hw_stats(adapter);
642 /* Tell the stack that the interface is not active */
643 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
644 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
646 adapter->led_dev = led_create(igb_led_func, adapter,
647 device_get_nameunit(dev));
650 ** Configure Interrupts
652 if ((adapter->msix > 1) && (igb_enable_msix))
653 error = igb_allocate_msix(adapter);
654 else /* MSI or Legacy */
655 error = igb_allocate_legacy(adapter);
660 igb_netmap_attach(adapter);
661 #endif /* DEV_NETMAP */
662 INIT_DEBUGOUT("igb_attach: end");
668 igb_free_transmit_structures(adapter);
669 igb_free_receive_structures(adapter);
670 igb_release_hw_control(adapter);
672 igb_free_pci_resources(adapter);
673 if (adapter->ifp != NULL)
674 if_free(adapter->ifp);
675 free(adapter->mta, M_DEVBUF);
676 IGB_CORE_LOCK_DESTROY(adapter);
681 /*********************************************************************
682 * Device removal routine
684 * The detach entry point is called when the driver is being removed.
685 * This routine stops the adapter and deallocates all the resources
686 * that were allocated for driver operation.
688 * return 0 on success, positive on failure
689 *********************************************************************/
692 igb_detach(device_t dev)
694 struct adapter *adapter = device_get_softc(dev);
695 struct ifnet *ifp = adapter->ifp;
697 INIT_DEBUGOUT("igb_detach: begin");
699 /* Make sure VLANS are not using driver */
700 if (adapter->ifp->if_vlantrunk != NULL) {
701 device_printf(dev,"Vlan in use, detach first\n");
705 ether_ifdetach(adapter->ifp);
707 if (adapter->led_dev != NULL)
708 led_destroy(adapter->led_dev);
710 #ifdef DEVICE_POLLING
711 if (ifp->if_capenable & IFCAP_POLLING)
712 ether_poll_deregister(ifp);
715 IGB_CORE_LOCK(adapter);
716 adapter->in_detach = 1;
718 IGB_CORE_UNLOCK(adapter);
720 e1000_phy_hw_reset(&adapter->hw);
722 /* Give control back to firmware */
723 igb_release_manageability(adapter);
724 igb_release_hw_control(adapter);
727 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
728 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
729 igb_enable_wakeup(dev);
732 /* Unregister VLAN events */
733 if (adapter->vlan_attach != NULL)
734 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
735 if (adapter->vlan_detach != NULL)
736 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
738 callout_drain(&adapter->timer);
741 netmap_detach(adapter->ifp);
742 #endif /* DEV_NETMAP */
743 igb_free_pci_resources(adapter);
744 bus_generic_detach(dev);
747 igb_free_transmit_structures(adapter);
748 igb_free_receive_structures(adapter);
749 if (adapter->mta != NULL)
750 free(adapter->mta, M_DEVBUF);
752 IGB_CORE_LOCK_DESTROY(adapter);
757 /*********************************************************************
759 * Shutdown entry point
761 **********************************************************************/
764 igb_shutdown(device_t dev)
766 return igb_suspend(dev);
770 * Suspend/resume device methods.
773 igb_suspend(device_t dev)
775 struct adapter *adapter = device_get_softc(dev);
777 IGB_CORE_LOCK(adapter);
781 igb_release_manageability(adapter);
782 igb_release_hw_control(adapter);
785 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
786 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
787 igb_enable_wakeup(dev);
790 IGB_CORE_UNLOCK(adapter);
792 return bus_generic_suspend(dev);
796 igb_resume(device_t dev)
798 struct adapter *adapter = device_get_softc(dev);
799 struct tx_ring *txr = adapter->tx_rings;
800 struct ifnet *ifp = adapter->ifp;
802 IGB_CORE_LOCK(adapter);
803 igb_init_locked(adapter);
804 igb_init_manageability(adapter);
806 if ((ifp->if_flags & IFF_UP) &&
807 (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
808 for (int i = 0; i < adapter->num_queues; i++, txr++) {
810 #ifndef IGB_LEGACY_TX
811 /* Process the stack queue only if not depleted */
812 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
813 !drbr_empty(ifp, txr->br))
814 igb_mq_start_locked(ifp, txr);
816 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
817 igb_start_locked(txr, ifp);
822 IGB_CORE_UNLOCK(adapter);
824 return bus_generic_resume(dev);
830 /*********************************************************************
831 * Transmit entry point
833 * igb_start is called by the stack to initiate a transmit.
834 * The driver will remain in this routine as long as there are
835 * packets to transmit and transmit resources are available.
836 * In case resources are not available stack is notified and
837 * the packet is requeued.
838 **********************************************************************/
841 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
843 struct adapter *adapter = ifp->if_softc;
846 IGB_TX_LOCK_ASSERT(txr);
848 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
851 if (!adapter->link_active)
854 /* Call cleanup if number of TX descriptors low */
855 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
858 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
859 if (txr->tx_avail <= IGB_MAX_SCATTER) {
860 txr->queue_status |= IGB_QUEUE_DEPLETED;
863 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
867 * Encapsulation can modify our pointer, and or make it
868 * NULL on failure. In that event, we can't requeue.
870 if (igb_xmit(txr, &m_head)) {
872 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
873 if (txr->tx_avail <= IGB_MAX_SCATTER)
874 txr->queue_status |= IGB_QUEUE_DEPLETED;
878 /* Send a copy of the frame to the BPF listener */
879 ETHER_BPF_MTAP(ifp, m_head);
881 /* Set watchdog on */
882 txr->watchdog_time = ticks;
883 txr->queue_status |= IGB_QUEUE_WORKING;
888 * Legacy TX driver routine, called from the
889 * stack, always uses tx[0], and spins for it.
890 * Should not be used with multiqueue tx
893 igb_start(struct ifnet *ifp)
895 struct adapter *adapter = ifp->if_softc;
896 struct tx_ring *txr = adapter->tx_rings;
898 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
900 igb_start_locked(txr, ifp);
906 #else /* ~IGB_LEGACY_TX */
909 ** Multiqueue Transmit Entry:
910 ** quick turnaround to the stack
914 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
916 struct adapter *adapter = ifp->if_softc;
917 struct igb_queue *que;
921 /* Which queue to use */
922 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
923 i = m->m_pkthdr.flowid % adapter->num_queues;
925 i = curcpu % adapter->num_queues;
926 txr = &adapter->tx_rings[i];
927 que = &adapter->queues[i];
929 err = drbr_enqueue(ifp, txr->br, m);
932 if (IGB_TX_TRYLOCK(txr)) {
933 igb_mq_start_locked(ifp, txr);
936 taskqueue_enqueue(que->tq, &txr->txq_task);
942 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
944 struct adapter *adapter = txr->adapter;
946 int err = 0, enq = 0;
948 IGB_TX_LOCK_ASSERT(txr);
950 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
951 adapter->link_active == 0)
954 /* Process the queue */
955 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
956 if ((err = igb_xmit(txr, &next)) != 0) {
958 /* It was freed, move forward */
959 drbr_advance(ifp, txr->br);
962 * Still have one left, it may not be
963 * the same since the transmit function
964 * may have changed it.
966 drbr_putback(ifp, txr->br, next);
970 drbr_advance(ifp, txr->br);
972 ifp->if_obytes += next->m_pkthdr.len;
973 if (next->m_flags & M_MCAST)
975 ETHER_BPF_MTAP(ifp, next);
976 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
980 /* Set the watchdog */
981 txr->queue_status |= IGB_QUEUE_WORKING;
982 txr->watchdog_time = ticks;
984 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
986 if (txr->tx_avail <= IGB_MAX_SCATTER)
987 txr->queue_status |= IGB_QUEUE_DEPLETED;
992 * Called from a taskqueue to drain queued transmit packets.
995 igb_deferred_mq_start(void *arg, int pending)
997 struct tx_ring *txr = arg;
998 struct adapter *adapter = txr->adapter;
999 struct ifnet *ifp = adapter->ifp;
1002 if (!drbr_empty(ifp, txr->br))
1003 igb_mq_start_locked(ifp, txr);
1008 ** Flush all ring buffers
1011 igb_qflush(struct ifnet *ifp)
1013 struct adapter *adapter = ifp->if_softc;
1014 struct tx_ring *txr = adapter->tx_rings;
1017 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1019 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1025 #endif /* ~IGB_LEGACY_TX */
1027 /*********************************************************************
1030 * igb_ioctl is called when the user wants to configure the
1033 * return 0 on success, positive on failure
1034 **********************************************************************/
1037 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1039 struct adapter *adapter = ifp->if_softc;
1040 struct ifreq *ifr = (struct ifreq *)data;
1041 #if defined(INET) || defined(INET6)
1042 struct ifaddr *ifa = (struct ifaddr *)data;
1044 bool avoid_reset = FALSE;
1047 if (adapter->in_detach)
1053 if (ifa->ifa_addr->sa_family == AF_INET)
1057 if (ifa->ifa_addr->sa_family == AF_INET6)
1061 ** Calling init results in link renegotiation,
1062 ** so we avoid doing it when possible.
1065 ifp->if_flags |= IFF_UP;
1066 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1069 if (!(ifp->if_flags & IFF_NOARP))
1070 arp_ifinit(ifp, ifa);
1073 error = ether_ioctl(ifp, command, data);
1079 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1081 IGB_CORE_LOCK(adapter);
1082 max_frame_size = 9234;
1083 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1085 IGB_CORE_UNLOCK(adapter);
1090 ifp->if_mtu = ifr->ifr_mtu;
1091 adapter->max_frame_size =
1092 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1093 igb_init_locked(adapter);
1094 IGB_CORE_UNLOCK(adapter);
1098 IOCTL_DEBUGOUT("ioctl rcv'd:\
1099 SIOCSIFFLAGS (Set Interface Flags)");
1100 IGB_CORE_LOCK(adapter);
1101 if (ifp->if_flags & IFF_UP) {
1102 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1103 if ((ifp->if_flags ^ adapter->if_flags) &
1104 (IFF_PROMISC | IFF_ALLMULTI)) {
1105 igb_disable_promisc(adapter);
1106 igb_set_promisc(adapter);
1109 igb_init_locked(adapter);
1111 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1113 adapter->if_flags = ifp->if_flags;
1114 IGB_CORE_UNLOCK(adapter);
1118 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1119 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1120 IGB_CORE_LOCK(adapter);
1121 igb_disable_intr(adapter);
1122 igb_set_multi(adapter);
1123 #ifdef DEVICE_POLLING
1124 if (!(ifp->if_capenable & IFCAP_POLLING))
1126 igb_enable_intr(adapter);
1127 IGB_CORE_UNLOCK(adapter);
1131 /* Check SOL/IDER usage */
1132 IGB_CORE_LOCK(adapter);
1133 if (e1000_check_reset_block(&adapter->hw)) {
1134 IGB_CORE_UNLOCK(adapter);
1135 device_printf(adapter->dev, "Media change is"
1136 " blocked due to SOL/IDER session.\n");
1139 IGB_CORE_UNLOCK(adapter);
1141 IOCTL_DEBUGOUT("ioctl rcv'd: \
1142 SIOCxIFMEDIA (Get/Set Interface Media)");
1143 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1149 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1151 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1152 #ifdef DEVICE_POLLING
1153 if (mask & IFCAP_POLLING) {
1154 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1155 error = ether_poll_register(igb_poll, ifp);
1158 IGB_CORE_LOCK(adapter);
1159 igb_disable_intr(adapter);
1160 ifp->if_capenable |= IFCAP_POLLING;
1161 IGB_CORE_UNLOCK(adapter);
1163 error = ether_poll_deregister(ifp);
1164 /* Enable interrupt even in error case */
1165 IGB_CORE_LOCK(adapter);
1166 igb_enable_intr(adapter);
1167 ifp->if_capenable &= ~IFCAP_POLLING;
1168 IGB_CORE_UNLOCK(adapter);
1172 if (mask & IFCAP_HWCSUM) {
1173 ifp->if_capenable ^= IFCAP_HWCSUM;
1176 if (mask & IFCAP_TSO4) {
1177 ifp->if_capenable ^= IFCAP_TSO4;
1180 if (mask & IFCAP_TSO6) {
1181 ifp->if_capenable ^= IFCAP_TSO6;
1184 if (mask & IFCAP_VLAN_HWTAGGING) {
1185 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1188 if (mask & IFCAP_VLAN_HWFILTER) {
1189 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1192 if (mask & IFCAP_VLAN_HWTSO) {
1193 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1196 if (mask & IFCAP_LRO) {
1197 ifp->if_capenable ^= IFCAP_LRO;
1200 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1202 VLAN_CAPABILITIES(ifp);
1207 error = ether_ioctl(ifp, command, data);
1215 /*********************************************************************
1218 * This routine is used in two ways. It is used by the stack as
1219 * init entry point in network interface structure. It is also used
1220 * by the driver as a hw/sw initialization routine to get to a
1223 * return 0 on success, positive on failure
1224 **********************************************************************/
1227 igb_init_locked(struct adapter *adapter)
1229 struct ifnet *ifp = adapter->ifp;
1230 device_t dev = adapter->dev;
1232 INIT_DEBUGOUT("igb_init: begin");
1234 IGB_CORE_LOCK_ASSERT(adapter);
1236 igb_disable_intr(adapter);
1237 callout_stop(&adapter->timer);
1239 /* Get the latest mac address, User can use a LAA */
1240 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1243 /* Put the address into the Receive Address Array */
1244 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1247 igb_update_link_status(adapter);
1249 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1251 /* Set hardware offload abilities */
1252 ifp->if_hwassist = 0;
1253 if (ifp->if_capenable & IFCAP_TXCSUM) {
1254 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1255 #if __FreeBSD_version >= 800000
1256 if ((adapter->hw.mac.type == e1000_82576) ||
1257 (adapter->hw.mac.type == e1000_82580))
1258 ifp->if_hwassist |= CSUM_SCTP;
1262 if (ifp->if_capenable & IFCAP_TSO)
1263 ifp->if_hwassist |= CSUM_TSO;
1265 /* Clear bad data from Rx FIFOs */
1266 e1000_rx_fifo_flush_82575(&adapter->hw);
1268 /* Configure for OS presence */
1269 igb_init_manageability(adapter);
1271 /* Prepare transmit descriptors and buffers */
1272 igb_setup_transmit_structures(adapter);
1273 igb_initialize_transmit_units(adapter);
1275 /* Setup Multicast table */
1276 igb_set_multi(adapter);
1279 ** Figure out the desired mbuf pool
1280 ** for doing jumbo/packetsplit
1282 if (adapter->max_frame_size <= 2048)
1283 adapter->rx_mbuf_sz = MCLBYTES;
1284 else if (adapter->max_frame_size <= 4096)
1285 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1287 adapter->rx_mbuf_sz = MJUM9BYTES;
1289 /* Prepare receive descriptors and buffers */
1290 if (igb_setup_receive_structures(adapter)) {
1291 device_printf(dev, "Could not setup receive structures\n");
1294 igb_initialize_receive_units(adapter);
1296 /* Enable VLAN support */
1297 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1298 igb_setup_vlan_hw_support(adapter);
1300 /* Don't lose promiscuous settings */
1301 igb_set_promisc(adapter);
1303 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1304 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1306 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1307 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1309 if (adapter->msix > 1) /* Set up queue routing */
1310 igb_configure_queues(adapter);
1312 /* this clears any pending interrupts */
1313 E1000_READ_REG(&adapter->hw, E1000_ICR);
1314 #ifdef DEVICE_POLLING
1316 * Only enable interrupts if we are not polling, make sure
1317 * they are off otherwise.
1319 if (ifp->if_capenable & IFCAP_POLLING)
1320 igb_disable_intr(adapter);
1322 #endif /* DEVICE_POLLING */
1324 igb_enable_intr(adapter);
1325 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1328 /* Set Energy Efficient Ethernet */
1329 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
1330 if (adapter->hw.mac.type == e1000_i354)
1331 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
1333 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
1340 struct adapter *adapter = arg;
1342 IGB_CORE_LOCK(adapter);
1343 igb_init_locked(adapter);
1344 IGB_CORE_UNLOCK(adapter);
1349 igb_handle_que(void *context, int pending)
1351 struct igb_queue *que = context;
1352 struct adapter *adapter = que->adapter;
1353 struct tx_ring *txr = que->txr;
1354 struct ifnet *ifp = adapter->ifp;
1356 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1359 more = igb_rxeof(que, adapter->rx_process_limit, NULL);
1363 #ifndef IGB_LEGACY_TX
1364 /* Process the stack queue only if not depleted */
1365 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1366 !drbr_empty(ifp, txr->br))
1367 igb_mq_start_locked(ifp, txr);
1369 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1370 igb_start_locked(txr, ifp);
1373 /* Do we need another? */
1375 taskqueue_enqueue(que->tq, &que->que_task);
1380 #ifdef DEVICE_POLLING
1381 if (ifp->if_capenable & IFCAP_POLLING)
1384 /* Reenable this interrupt */
1386 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1388 igb_enable_intr(adapter);
1391 /* Deal with link in a sleepable context */
1393 igb_handle_link(void *context, int pending)
1395 struct adapter *adapter = context;
1397 IGB_CORE_LOCK(adapter);
1398 igb_handle_link_locked(adapter);
1399 IGB_CORE_UNLOCK(adapter);
1403 igb_handle_link_locked(struct adapter *adapter)
1405 struct tx_ring *txr = adapter->tx_rings;
1406 struct ifnet *ifp = adapter->ifp;
1408 IGB_CORE_LOCK_ASSERT(adapter);
1409 adapter->hw.mac.get_link_status = 1;
1410 igb_update_link_status(adapter);
1411 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
1412 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1414 #ifndef IGB_LEGACY_TX
1415 /* Process the stack queue only if not depleted */
1416 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1417 !drbr_empty(ifp, txr->br))
1418 igb_mq_start_locked(ifp, txr);
1420 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1421 igb_start_locked(txr, ifp);
1428 /*********************************************************************
1430 * MSI/Legacy Deferred
1431 * Interrupt Service routine
1433 *********************************************************************/
1435 igb_irq_fast(void *arg)
1437 struct adapter *adapter = arg;
1438 struct igb_queue *que = adapter->queues;
1442 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1445 if (reg_icr == 0xffffffff)
1446 return FILTER_STRAY;
1448 /* Definitely not our interrupt. */
1450 return FILTER_STRAY;
1452 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1453 return FILTER_STRAY;
1456 * Mask interrupts until the taskqueue is finished running. This is
1457 * cheap, just assume that it is needed. This also works around the
1458 * MSI message reordering errata on certain systems.
1460 igb_disable_intr(adapter);
1461 taskqueue_enqueue(que->tq, &que->que_task);
1463 /* Link status change */
1464 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1465 taskqueue_enqueue(que->tq, &adapter->link_task);
1467 if (reg_icr & E1000_ICR_RXO)
1468 adapter->rx_overruns++;
1469 return FILTER_HANDLED;
1472 #ifdef DEVICE_POLLING
1473 #if __FreeBSD_version >= 800000
1474 #define POLL_RETURN_COUNT(a) (a)
1477 #define POLL_RETURN_COUNT(a)
1480 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1482 struct adapter *adapter = ifp->if_softc;
1483 struct igb_queue *que;
1484 struct tx_ring *txr;
1485 u32 reg_icr, rx_done = 0;
1486 u32 loop = IGB_MAX_LOOP;
1489 IGB_CORE_LOCK(adapter);
1490 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1491 IGB_CORE_UNLOCK(adapter);
1492 return POLL_RETURN_COUNT(rx_done);
1495 if (cmd == POLL_AND_CHECK_STATUS) {
1496 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1497 /* Link status change */
1498 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1499 igb_handle_link_locked(adapter);
1501 if (reg_icr & E1000_ICR_RXO)
1502 adapter->rx_overruns++;
1504 IGB_CORE_UNLOCK(adapter);
1506 for (int i = 0; i < adapter->num_queues; i++) {
1507 que = &adapter->queues[i];
1510 igb_rxeof(que, count, &rx_done);
1514 more = igb_txeof(txr);
1515 } while (loop-- && more);
1516 #ifndef IGB_LEGACY_TX
1517 if (!drbr_empty(ifp, txr->br))
1518 igb_mq_start_locked(ifp, txr);
1520 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1521 igb_start_locked(txr, ifp);
1526 return POLL_RETURN_COUNT(rx_done);
1528 #endif /* DEVICE_POLLING */
1530 /*********************************************************************
1532 * MSIX Que Interrupt Service routine
1534 **********************************************************************/
1536 igb_msix_que(void *arg)
1538 struct igb_queue *que = arg;
1539 struct adapter *adapter = que->adapter;
1540 struct ifnet *ifp = adapter->ifp;
1541 struct tx_ring *txr = que->txr;
1542 struct rx_ring *rxr = que->rxr;
1546 /* Ignore spurious interrupts */
1547 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1550 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1555 #ifndef IGB_LEGACY_TX
1556 /* Process the stack queue only if not depleted */
1557 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1558 !drbr_empty(ifp, txr->br))
1559 igb_mq_start_locked(ifp, txr);
1561 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1562 igb_start_locked(txr, ifp);
1566 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
1568 if (adapter->enable_aim == FALSE)
1571 ** Do Adaptive Interrupt Moderation:
1572 ** - Write out last calculated setting
1573 ** - Calculate based on average size over
1574 ** the last interval.
1576 if (que->eitr_setting)
1577 E1000_WRITE_REG(&adapter->hw,
1578 E1000_EITR(que->msix), que->eitr_setting);
1580 que->eitr_setting = 0;
1582 /* Idle, do nothing */
1583 if ((txr->bytes == 0) && (rxr->bytes == 0))
1586 /* Used half Default if sub-gig */
1587 if (adapter->link_speed != 1000)
1588 newitr = IGB_DEFAULT_ITR / 2;
1590 if ((txr->bytes) && (txr->packets))
1591 newitr = txr->bytes/txr->packets;
1592 if ((rxr->bytes) && (rxr->packets))
1593 newitr = max(newitr,
1594 (rxr->bytes / rxr->packets));
1595 newitr += 24; /* account for hardware frame, crc */
1596 /* set an upper boundary */
1597 newitr = min(newitr, 3000);
1598 /* Be nice to the mid range */
1599 if ((newitr > 300) && (newitr < 1200))
1600 newitr = (newitr / 3);
1602 newitr = (newitr / 2);
1604 newitr &= 0x7FFC; /* Mask invalid bits */
1605 if (adapter->hw.mac.type == e1000_82575)
1606 newitr |= newitr << 16;
1608 newitr |= E1000_EITR_CNT_IGNR;
1610 /* save for next interrupt */
1611 que->eitr_setting = newitr;
1620 /* Schedule a clean task if needed*/
1622 taskqueue_enqueue(que->tq, &que->que_task);
1624 /* Reenable this interrupt */
1625 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1630 /*********************************************************************
1632 * MSIX Link Interrupt Service routine
1634 **********************************************************************/
1637 igb_msix_link(void *arg)
1639 struct adapter *adapter = arg;
1642 ++adapter->link_irq;
1643 icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1644 if (!(icr & E1000_ICR_LSC))
1646 igb_handle_link(adapter, 0);
1650 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1651 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1656 /*********************************************************************
1658 * Media Ioctl callback
1660 * This routine is called whenever the user queries the status of
1661 * the interface using ifconfig.
1663 **********************************************************************/
1665 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1667 struct adapter *adapter = ifp->if_softc;
1669 INIT_DEBUGOUT("igb_media_status: begin");
1671 IGB_CORE_LOCK(adapter);
1672 igb_update_link_status(adapter);
1674 ifmr->ifm_status = IFM_AVALID;
1675 ifmr->ifm_active = IFM_ETHER;
1677 if (!adapter->link_active) {
1678 IGB_CORE_UNLOCK(adapter);
1682 ifmr->ifm_status |= IFM_ACTIVE;
1684 switch (adapter->link_speed) {
1686 ifmr->ifm_active |= IFM_10_T;
1690 ** Support for 100Mb SFP - these are Fiber
1691 ** but the media type appears as serdes
1693 if (adapter->hw.phy.media_type ==
1694 e1000_media_type_internal_serdes)
1695 ifmr->ifm_active |= IFM_100_FX;
1697 ifmr->ifm_active |= IFM_100_TX;
1700 ifmr->ifm_active |= IFM_1000_T;
1703 ifmr->ifm_active |= IFM_2500_SX;
1707 if (adapter->link_duplex == FULL_DUPLEX)
1708 ifmr->ifm_active |= IFM_FDX;
1710 ifmr->ifm_active |= IFM_HDX;
1712 IGB_CORE_UNLOCK(adapter);
1715 /*********************************************************************
1717 * Media Ioctl callback
1719 * This routine is called when the user changes speed/duplex using
1720 * media/mediopt option with ifconfig.
1722 **********************************************************************/
1724 igb_media_change(struct ifnet *ifp)
1726 struct adapter *adapter = ifp->if_softc;
1727 struct ifmedia *ifm = &adapter->media;
1729 INIT_DEBUGOUT("igb_media_change: begin");
1731 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1734 IGB_CORE_LOCK(adapter);
1735 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1737 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1738 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1743 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1744 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1747 adapter->hw.mac.autoneg = FALSE;
1748 adapter->hw.phy.autoneg_advertised = 0;
1749 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1750 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1752 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1755 adapter->hw.mac.autoneg = FALSE;
1756 adapter->hw.phy.autoneg_advertised = 0;
1757 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1758 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1760 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1763 device_printf(adapter->dev, "Unsupported media type\n");
1766 igb_init_locked(adapter);
1767 IGB_CORE_UNLOCK(adapter);
1773 /*********************************************************************
1775 * This routine maps the mbufs to Advanced TX descriptors.
1777 **********************************************************************/
1779 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1781 struct adapter *adapter = txr->adapter;
1782 u32 olinfo_status = 0, cmd_type_len;
1783 int i, j, error, nsegs;
1786 struct mbuf *m_head;
1787 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1789 struct igb_tx_buf *txbuf;
1790 union e1000_adv_tx_desc *txd = NULL;
1794 /* Basic descriptor defines */
1795 cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
1796 E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
1798 if (m_head->m_flags & M_VLANTAG)
1799 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1802 * Important to capture the first descriptor
1803 * used because it will contain the index of
1804 * the one we tell the hardware to report back
1806 first = txr->next_avail_desc;
1807 txbuf = &txr->tx_buffers[first];
1811 * Map the packet for DMA.
1814 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1815 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1817 if (__predict_false(error)) {
1822 /* Try it again? - one try */
1823 if (remap == TRUE) {
1825 m = m_collapse(*m_headp, M_NOWAIT,
1828 adapter->mbuf_defrag_failed++;
1838 txr->no_tx_dma_setup++;
1845 /* Make certain there are enough descriptors */
1846 if (nsegs > txr->tx_avail - 2) {
1847 txr->no_desc_avail++;
1848 bus_dmamap_unload(txr->txtag, map);
1854 ** Set up the appropriate offload context
1855 ** this will consume the first descriptor
1857 error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1858 if (__predict_false(error)) {
1864 /* 82575 needs the queue index added */
1865 if (adapter->hw.mac.type == e1000_82575)
1866 olinfo_status |= txr->me << 4;
1868 i = txr->next_avail_desc;
1869 for (j = 0; j < nsegs; j++) {
1873 txbuf = &txr->tx_buffers[i];
1874 txd = &txr->tx_base[i];
1875 seglen = segs[j].ds_len;
1876 segaddr = htole64(segs[j].ds_addr);
1878 txd->read.buffer_addr = segaddr;
1879 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
1880 cmd_type_len | seglen);
1881 txd->read.olinfo_status = htole32(olinfo_status);
1883 if (++i == txr->num_desc)
1887 txd->read.cmd_type_len |=
1888 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1889 txr->tx_avail -= nsegs;
1890 txr->next_avail_desc = i;
1892 txbuf->m_head = m_head;
1894 ** Here we swap the map so the last descriptor,
1895 ** which gets the completion interrupt has the
1896 ** real map, and the first descriptor gets the
1897 ** unused map from this descriptor.
1899 txr->tx_buffers[first].map = txbuf->map;
1901 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1903 /* Set the EOP descriptor that will be marked done */
1904 txbuf = &txr->tx_buffers[first];
1907 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1908 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1910 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1911 * hardware that this frame is available to transmit.
1913 ++txr->total_packets;
1914 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1919 igb_set_promisc(struct adapter *adapter)
1921 struct ifnet *ifp = adapter->ifp;
1922 struct e1000_hw *hw = &adapter->hw;
1925 if (adapter->vf_ifp) {
1926 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
1930 reg = E1000_READ_REG(hw, E1000_RCTL);
1931 if (ifp->if_flags & IFF_PROMISC) {
1932 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1933 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1934 } else if (ifp->if_flags & IFF_ALLMULTI) {
1935 reg |= E1000_RCTL_MPE;
1936 reg &= ~E1000_RCTL_UPE;
1937 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1942 igb_disable_promisc(struct adapter *adapter)
1944 struct e1000_hw *hw = &adapter->hw;
1945 struct ifnet *ifp = adapter->ifp;
1949 if (adapter->vf_ifp) {
1950 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
1953 reg = E1000_READ_REG(hw, E1000_RCTL);
1954 reg &= (~E1000_RCTL_UPE);
1955 if (ifp->if_flags & IFF_ALLMULTI)
1956 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1958 struct ifmultiaddr *ifma;
1959 #if __FreeBSD_version < 800000
1962 if_maddr_rlock(ifp);
1964 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1965 if (ifma->ifma_addr->sa_family != AF_LINK)
1967 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1971 #if __FreeBSD_version < 800000
1972 IF_ADDR_UNLOCK(ifp);
1974 if_maddr_runlock(ifp);
1977 /* Don't disable if in MAX groups */
1978 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1979 reg &= (~E1000_RCTL_MPE);
1980 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1984 /*********************************************************************
1987 * This routine is called whenever multicast address list is updated.
1989 **********************************************************************/
1992 igb_set_multi(struct adapter *adapter)
1994 struct ifnet *ifp = adapter->ifp;
1995 struct ifmultiaddr *ifma;
2001 IOCTL_DEBUGOUT("igb_set_multi: begin");
2004 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
2005 MAX_NUM_MULTICAST_ADDRESSES);
2007 #if __FreeBSD_version < 800000
2010 if_maddr_rlock(ifp);
2012 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2013 if (ifma->ifma_addr->sa_family != AF_LINK)
2016 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2019 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2020 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2023 #if __FreeBSD_version < 800000
2024 IF_ADDR_UNLOCK(ifp);
2026 if_maddr_runlock(ifp);
2029 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2030 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2031 reg_rctl |= E1000_RCTL_MPE;
2032 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2034 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2038 /*********************************************************************
2040 * This routine checks for link status,
2041 * updates statistics, and does the watchdog.
2043 **********************************************************************/
2046 igb_local_timer(void *arg)
2048 struct adapter *adapter = arg;
2049 device_t dev = adapter->dev;
2050 struct ifnet *ifp = adapter->ifp;
2051 struct tx_ring *txr = adapter->tx_rings;
2052 struct igb_queue *que = adapter->queues;
2053 int hung = 0, busy = 0;
2056 IGB_CORE_LOCK_ASSERT(adapter);
2058 igb_update_link_status(adapter);
2059 igb_update_stats_counters(adapter);
2062 ** Check the TX queues status
2063 ** - central locked handling of OACTIVE
2064 ** - watchdog only if all queues show hung
2066 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2067 if ((txr->queue_status & IGB_QUEUE_HUNG) &&
2068 (adapter->pause_frames == 0))
2070 if (txr->queue_status & IGB_QUEUE_DEPLETED)
2072 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
2073 taskqueue_enqueue(que->tq, &que->que_task);
2075 if (hung == adapter->num_queues)
2077 if (busy == adapter->num_queues)
2078 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2079 else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
2080 (busy < adapter->num_queues))
2081 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2083 adapter->pause_frames = 0;
2084 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
2085 #ifndef DEVICE_POLLING
2086 /* Schedule all queue interrupts - deadlock protection */
2087 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
2092 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2093 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2094 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2095 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2096 device_printf(dev,"TX(%d) desc avail = %d,"
2097 "Next TX to Clean = %d\n",
2098 txr->me, txr->tx_avail, txr->next_to_clean);
2099 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2100 adapter->watchdog_events++;
2101 igb_init_locked(adapter);
2105 igb_update_link_status(struct adapter *adapter)
2107 struct e1000_hw *hw = &adapter->hw;
2108 struct e1000_fc_info *fc = &hw->fc;
2109 struct ifnet *ifp = adapter->ifp;
2110 device_t dev = adapter->dev;
2111 struct tx_ring *txr = adapter->tx_rings;
2112 u32 link_check, thstat, ctrl;
2113 char *flowctl = NULL;
2115 link_check = thstat = ctrl = 0;
2117 /* Get the cached link value or read for real */
2118 switch (hw->phy.media_type) {
2119 case e1000_media_type_copper:
2120 if (hw->mac.get_link_status) {
2121 /* Do the work to read phy */
2122 e1000_check_for_link(hw);
2123 link_check = !hw->mac.get_link_status;
2127 case e1000_media_type_fiber:
2128 e1000_check_for_link(hw);
2129 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2132 case e1000_media_type_internal_serdes:
2133 e1000_check_for_link(hw);
2134 link_check = adapter->hw.mac.serdes_has_link;
2136 /* VF device is type_unknown */
2137 case e1000_media_type_unknown:
2138 e1000_check_for_link(hw);
2139 link_check = !hw->mac.get_link_status;
2145 /* Check for thermal downshift or shutdown */
2146 if (hw->mac.type == e1000_i350) {
2147 thstat = E1000_READ_REG(hw, E1000_THSTAT);
2148 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
2151 /* Get the flow control for display */
2152 switch (fc->current_mode) {
2153 case e1000_fc_rx_pause:
2156 case e1000_fc_tx_pause:
2168 /* Now we check if a transition has happened */
2169 if (link_check && (adapter->link_active == 0)) {
2170 e1000_get_speed_and_duplex(&adapter->hw,
2171 &adapter->link_speed, &adapter->link_duplex);
2173 device_printf(dev, "Link is up %d Mbps %s,"
2174 " Flow Control: %s\n",
2175 adapter->link_speed,
2176 ((adapter->link_duplex == FULL_DUPLEX) ?
2177 "Full Duplex" : "Half Duplex"), flowctl);
2178 adapter->link_active = 1;
2179 ifp->if_baudrate = adapter->link_speed * 1000000;
2180 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2181 (thstat & E1000_THSTAT_LINK_THROTTLE))
2182 device_printf(dev, "Link: thermal downshift\n");
2183 /* Delay Link Up for Phy update */
2184 if (((hw->mac.type == e1000_i210) ||
2185 (hw->mac.type == e1000_i211)) &&
2186 (hw->phy.id == I210_I_PHY_ID))
2187 msec_delay(I210_LINK_DELAY);
2188 /* Reset if the media type changed. */
2189 if (hw->dev_spec._82575.media_changed) {
2190 hw->dev_spec._82575.media_changed = false;
2191 adapter->flags |= IGB_MEDIA_RESET;
2194 /* This can sleep */
2195 if_link_state_change(ifp, LINK_STATE_UP);
2196 } else if (!link_check && (adapter->link_active == 1)) {
2197 ifp->if_baudrate = adapter->link_speed = 0;
2198 adapter->link_duplex = 0;
2200 device_printf(dev, "Link is Down\n");
2201 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2202 (thstat & E1000_THSTAT_PWR_DOWN))
2203 device_printf(dev, "Link: thermal shutdown\n");
2204 adapter->link_active = 0;
2205 /* This can sleep */
2206 if_link_state_change(ifp, LINK_STATE_DOWN);
2207 /* Reset queue state */
2208 for (int i = 0; i < adapter->num_queues; i++, txr++)
2209 txr->queue_status = IGB_QUEUE_IDLE;
2213 /*********************************************************************
2215 * This routine disables all traffic on the adapter by issuing a
2216 * global reset on the MAC and deallocates TX/RX buffers.
2218 **********************************************************************/
2223 struct adapter *adapter = arg;
2224 struct ifnet *ifp = adapter->ifp;
2225 struct tx_ring *txr = adapter->tx_rings;
2227 IGB_CORE_LOCK_ASSERT(adapter);
2229 INIT_DEBUGOUT("igb_stop: begin");
2231 igb_disable_intr(adapter);
2233 callout_stop(&adapter->timer);
2235 /* Tell the stack that the interface is no longer active */
2236 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2237 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2239 /* Disarm watchdog timer. */
2240 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2242 txr->queue_status = IGB_QUEUE_IDLE;
2246 e1000_reset_hw(&adapter->hw);
2247 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2249 e1000_led_off(&adapter->hw);
2250 e1000_cleanup_led(&adapter->hw);
2254 /*********************************************************************
2256 * Determine hardware revision.
2258 **********************************************************************/
2260 igb_identify_hardware(struct adapter *adapter)
2262 device_t dev = adapter->dev;
2264 /* Make sure our PCI config space has the necessary stuff set */
2265 pci_enable_busmaster(dev);
2266 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2268 /* Save off the information about this board */
2269 adapter->hw.vendor_id = pci_get_vendor(dev);
2270 adapter->hw.device_id = pci_get_device(dev);
2271 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2272 adapter->hw.subsystem_vendor_id =
2273 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2274 adapter->hw.subsystem_device_id =
2275 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2277 /* Set MAC type early for PCI setup */
2278 e1000_set_mac_type(&adapter->hw);
2280 /* Are we a VF device? */
2281 if ((adapter->hw.mac.type == e1000_vfadapt) ||
2282 (adapter->hw.mac.type == e1000_vfadapt_i350))
2283 adapter->vf_ifp = 1;
2285 adapter->vf_ifp = 0;
2289 igb_allocate_pci_resources(struct adapter *adapter)
2291 device_t dev = adapter->dev;
2295 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2297 if (adapter->pci_mem == NULL) {
2298 device_printf(dev, "Unable to allocate bus resource: memory\n");
2301 adapter->osdep.mem_bus_space_tag =
2302 rman_get_bustag(adapter->pci_mem);
2303 adapter->osdep.mem_bus_space_handle =
2304 rman_get_bushandle(adapter->pci_mem);
2305 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2307 adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2309 /* This will setup either MSI/X or MSI */
2310 adapter->msix = igb_setup_msix(adapter);
2311 adapter->hw.back = &adapter->osdep;
2316 /*********************************************************************
2318 * Setup the Legacy or MSI Interrupt handler
2320 **********************************************************************/
2322 igb_allocate_legacy(struct adapter *adapter)
2324 device_t dev = adapter->dev;
2325 struct igb_queue *que = adapter->queues;
2326 #ifndef IGB_LEGACY_TX
2327 struct tx_ring *txr = adapter->tx_rings;
2331 /* Turn off all interrupts */
2332 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2335 if (adapter->msix == 1)
2338 /* We allocate a single interrupt resource */
2339 adapter->res = bus_alloc_resource_any(dev,
2340 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2341 if (adapter->res == NULL) {
2342 device_printf(dev, "Unable to allocate bus resource: "
2347 #ifndef IGB_LEGACY_TX
2348 TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
2352 * Try allocating a fast interrupt and the associated deferred
2353 * processing contexts.
2355 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2356 /* Make tasklet for deferred link handling */
2357 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2358 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2359 taskqueue_thread_enqueue, &que->tq);
2360 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
2361 device_get_nameunit(adapter->dev));
2362 if ((error = bus_setup_intr(dev, adapter->res,
2363 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2364 adapter, &adapter->tag)) != 0) {
2365 device_printf(dev, "Failed to register fast interrupt "
2366 "handler: %d\n", error);
2367 taskqueue_free(que->tq);
2376 /*********************************************************************
2378 * Setup the MSIX Queue Interrupt handlers:
2380 **********************************************************************/
2382 igb_allocate_msix(struct adapter *adapter)
2384 device_t dev = adapter->dev;
2385 struct igb_queue *que = adapter->queues;
2386 int error, rid, vector = 0;
2388 /* Be sure to start with all interrupts disabled */
2389 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
2390 E1000_WRITE_FLUSH(&adapter->hw);
2392 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2394 que->res = bus_alloc_resource_any(dev,
2395 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2396 if (que->res == NULL) {
2398 "Unable to allocate bus resource: "
2399 "MSIX Queue Interrupt\n");
2402 error = bus_setup_intr(dev, que->res,
2403 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2404 igb_msix_que, que, &que->tag);
2407 device_printf(dev, "Failed to register Queue handler");
2410 #if __FreeBSD_version >= 800504
2411 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2414 if (adapter->hw.mac.type == e1000_82575)
2415 que->eims = E1000_EICR_TX_QUEUE0 << i;
2417 que->eims = 1 << vector;
2419 ** Bind the msix vector, and thus the
2420 ** rings to the corresponding cpu.
2422 if (adapter->num_queues > 1) {
2423 if (igb_last_bind_cpu < 0)
2424 igb_last_bind_cpu = CPU_FIRST();
2425 bus_bind_intr(dev, que->res, igb_last_bind_cpu);
2427 "Bound queue %d to cpu %d\n",
2428 i,igb_last_bind_cpu);
2429 igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
2431 #ifndef IGB_LEGACY_TX
2432 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
2435 /* Make tasklet for deferred handling */
2436 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2437 que->tq = taskqueue_create("igb_que", M_NOWAIT,
2438 taskqueue_thread_enqueue, &que->tq);
2439 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2440 device_get_nameunit(adapter->dev));
2445 adapter->res = bus_alloc_resource_any(dev,
2446 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2447 if (adapter->res == NULL) {
2449 "Unable to allocate bus resource: "
2450 "MSIX Link Interrupt\n");
2453 if ((error = bus_setup_intr(dev, adapter->res,
2454 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2455 igb_msix_link, adapter, &adapter->tag)) != 0) {
2456 device_printf(dev, "Failed to register Link handler");
2459 #if __FreeBSD_version >= 800504
2460 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2462 adapter->linkvec = vector;
2469 igb_configure_queues(struct adapter *adapter)
2471 struct e1000_hw *hw = &adapter->hw;
2472 struct igb_queue *que;
2473 u32 tmp, ivar = 0, newitr = 0;
2475 /* First turn on RSS capability */
2476 if (adapter->hw.mac.type != e1000_82575)
2477 E1000_WRITE_REG(hw, E1000_GPIE,
2478 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2479 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2482 switch (adapter->hw.mac.type) {
2489 case e1000_vfadapt_i350:
2491 for (int i = 0; i < adapter->num_queues; i++) {
2493 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2494 que = &adapter->queues[i];
2497 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2500 ivar |= que->msix | E1000_IVAR_VALID;
2502 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2505 for (int i = 0; i < adapter->num_queues; i++) {
2507 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2508 que = &adapter->queues[i];
2511 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2514 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2516 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2517 adapter->que_mask |= que->eims;
2520 /* And for the link interrupt */
2521 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2522 adapter->link_mask = 1 << adapter->linkvec;
2523 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2527 for (int i = 0; i < adapter->num_queues; i++) {
2528 u32 index = i & 0x7; /* Each IVAR has two entries */
2529 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2530 que = &adapter->queues[i];
2533 ivar |= que->msix | E1000_IVAR_VALID;
2536 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2538 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2539 adapter->que_mask |= que->eims;
2542 for (int i = 0; i < adapter->num_queues; i++) {
2543 u32 index = i & 0x7; /* Each IVAR has two entries */
2544 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2545 que = &adapter->queues[i];
2548 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2551 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2553 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2554 adapter->que_mask |= que->eims;
2557 /* And for the link interrupt */
2558 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2559 adapter->link_mask = 1 << adapter->linkvec;
2560 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2564 /* enable MSI-X support*/
2565 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2566 tmp |= E1000_CTRL_EXT_PBA_CLR;
2567 /* Auto-Mask interrupts upon ICR read. */
2568 tmp |= E1000_CTRL_EXT_EIAME;
2569 tmp |= E1000_CTRL_EXT_IRCA;
2570 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2573 for (int i = 0; i < adapter->num_queues; i++) {
2574 que = &adapter->queues[i];
2575 tmp = E1000_EICR_RX_QUEUE0 << i;
2576 tmp |= E1000_EICR_TX_QUEUE0 << i;
2578 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2580 adapter->que_mask |= que->eims;
2584 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2586 adapter->link_mask |= E1000_EIMS_OTHER;
2591 /* Set the starting interrupt rate */
2592 if (igb_max_interrupt_rate > 0)
2593 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
2595 if (hw->mac.type == e1000_82575)
2596 newitr |= newitr << 16;
2598 newitr |= E1000_EITR_CNT_IGNR;
2600 for (int i = 0; i < adapter->num_queues; i++) {
2601 que = &adapter->queues[i];
2602 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2610 igb_free_pci_resources(struct adapter *adapter)
2612 struct igb_queue *que = adapter->queues;
2613 device_t dev = adapter->dev;
2617 ** There is a slight possibility of a failure mode
2618 ** in attach that will result in entering this function
2619 ** before interrupt resources have been initialized, and
2620 ** in that case we do not want to execute the loops below
2621 ** We can detect this reliably by the state of the adapter
2624 if (adapter->res == NULL)
2628 * First release all the interrupt resources:
2630 for (int i = 0; i < adapter->num_queues; i++, que++) {
2631 rid = que->msix + 1;
2632 if (que->tag != NULL) {
2633 bus_teardown_intr(dev, que->res, que->tag);
2636 if (que->res != NULL)
2637 bus_release_resource(dev,
2638 SYS_RES_IRQ, rid, que->res);
2641 /* Clean the Legacy or Link interrupt last */
2642 if (adapter->linkvec) /* we are doing MSIX */
2643 rid = adapter->linkvec + 1;
2645 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2647 que = adapter->queues;
2648 if (adapter->tag != NULL) {
2649 taskqueue_drain(que->tq, &adapter->link_task);
2650 bus_teardown_intr(dev, adapter->res, adapter->tag);
2651 adapter->tag = NULL;
2653 if (adapter->res != NULL)
2654 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2656 for (int i = 0; i < adapter->num_queues; i++, que++) {
2657 if (que->tq != NULL) {
2658 #ifndef IGB_LEGACY_TX
2659 taskqueue_drain(que->tq, &que->txr->txq_task);
2661 taskqueue_drain(que->tq, &que->que_task);
2662 taskqueue_free(que->tq);
2667 pci_release_msi(dev);
2669 if (adapter->msix_mem != NULL)
2670 bus_release_resource(dev, SYS_RES_MEMORY,
2671 adapter->memrid, adapter->msix_mem);
2673 if (adapter->pci_mem != NULL)
2674 bus_release_resource(dev, SYS_RES_MEMORY,
2675 PCIR_BAR(0), adapter->pci_mem);
2680 * Setup Either MSI/X or MSI
2683 igb_setup_msix(struct adapter *adapter)
2685 device_t dev = adapter->dev;
2686 int bar, want, queues, msgs, maxqueues;
2688 /* tuneable override */
2689 if (igb_enable_msix == 0)
2692 /* First try MSI/X */
2693 msgs = pci_msix_count(dev);
2697 ** Some new devices, as with ixgbe, now may
2698 ** use a different BAR, so we need to keep
2699 ** track of which is used.
2701 adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
2702 bar = pci_read_config(dev, adapter->memrid, 4);
2703 if (bar == 0) /* use next bar */
2704 adapter->memrid += 4;
2705 adapter->msix_mem = bus_alloc_resource_any(dev,
2706 SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
2707 if (adapter->msix_mem == NULL) {
2708 /* May not be enabled */
2709 device_printf(adapter->dev,
2710 "Unable to map MSIX table \n");
2714 /* Figure out a reasonable auto config value */
2715 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2717 /* Manual override */
2718 if (igb_num_queues != 0)
2719 queues = igb_num_queues;
2721 /* Sanity check based on HW */
2722 switch (adapter->hw.mac.type) {
2738 default: /* VF interfaces */
2742 if (queues > maxqueues)
2745 /* Manual override */
2746 if (igb_num_queues != 0)
2747 queues = igb_num_queues;
2750 ** One vector (RX/TX pair) per queue
2751 ** plus an additional for Link interrupt
2757 device_printf(adapter->dev,
2758 "MSIX Configuration Problem, "
2759 "%d vectors configured, but %d queues wanted!\n",
2763 if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2764 device_printf(adapter->dev,
2765 "Using MSIX interrupts with %d vectors\n", msgs);
2766 adapter->num_queues = queues;
2770 ** If MSIX alloc failed or provided us with
2771 ** less than needed, free and fall through to MSI
2773 pci_release_msi(dev);
2776 if (adapter->msix_mem != NULL) {
2777 bus_release_resource(dev, SYS_RES_MEMORY,
2778 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2779 adapter->msix_mem = NULL;
2782 if (pci_alloc_msi(dev, &msgs) == 0) {
2783 device_printf(adapter->dev," Using an MSI interrupt\n");
2786 device_printf(adapter->dev," Using a Legacy interrupt\n");
2790 /*********************************************************************
2792 * Initialize the DMA Coalescing feature
2794 **********************************************************************/
2796 igb_init_dmac(struct adapter *adapter, u32 pba)
2798 device_t dev = adapter->dev;
2799 struct e1000_hw *hw = &adapter->hw;
2800 u32 dmac, reg = ~E1000_DMACR_DMAC_EN;
2803 if (hw->mac.type == e1000_i211)
2806 if (hw->mac.type > e1000_82580) {
2808 if (adapter->dmac == 0) { /* Disabling it */
2809 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2812 device_printf(dev, "DMA Coalescing enabled\n");
2814 /* Set starting threshold */
2815 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
2817 hwm = 64 * pba - adapter->max_frame_size / 16;
2818 if (hwm < 64 * (pba - 6))
2819 hwm = 64 * (pba - 6);
2820 reg = E1000_READ_REG(hw, E1000_FCRTC);
2821 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
2822 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
2823 & E1000_FCRTC_RTH_COAL_MASK);
2824 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
2827 dmac = pba - adapter->max_frame_size / 512;
2828 if (dmac < pba - 10)
2830 reg = E1000_READ_REG(hw, E1000_DMACR);
2831 reg &= ~E1000_DMACR_DMACTHR_MASK;
2832 reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
2833 & E1000_DMACR_DMACTHR_MASK);
2835 /* transition to L0x or L1 if available..*/
2836 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
2838 /* Check if status is 2.5Gb backplane connection
2839 * before configuration of watchdog timer, which is
2840 * in msec values in 12.8usec intervals
2841 * watchdog timer= msec values in 32usec intervals
2842 * for non 2.5Gb connection
2844 if (hw->mac.type == e1000_i354) {
2845 int status = E1000_READ_REG(hw, E1000_STATUS);
2846 if ((status & E1000_STATUS_2P5_SKU) &&
2847 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2848 reg |= ((adapter->dmac * 5) >> 6);
2850 reg |= (adapter->dmac >> 5);
2852 reg |= (adapter->dmac >> 5);
2855 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2857 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
2859 /* Set the interval before transition */
2860 reg = E1000_READ_REG(hw, E1000_DMCTLX);
2861 if (hw->mac.type == e1000_i350)
2862 reg |= IGB_DMCTLX_DCFLUSH_DIS;
2864 ** in 2.5Gb connection, TTLX unit is 0.4 usec
2865 ** which is 0x4*2 = 0xA. But delay is still 4 usec
2867 if (hw->mac.type == e1000_i354) {
2868 int status = E1000_READ_REG(hw, E1000_STATUS);
2869 if ((status & E1000_STATUS_2P5_SKU) &&
2870 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2878 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
2880 /* free space in tx packet buffer to wake from DMA coal */
2881 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
2882 (2 * adapter->max_frame_size)) >> 6);
2884 /* make low power state decision controlled by DMA coal */
2885 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2886 reg &= ~E1000_PCIEMISC_LX_DECISION;
2887 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
2889 } else if (hw->mac.type == e1000_82580) {
2890 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2891 E1000_WRITE_REG(hw, E1000_PCIEMISC,
2892 reg & ~E1000_PCIEMISC_LX_DECISION);
2893 E1000_WRITE_REG(hw, E1000_DMACR, 0);
2898 /*********************************************************************
2900 * Set up an fresh starting state
2902 **********************************************************************/
2904 igb_reset(struct adapter *adapter)
2906 device_t dev = adapter->dev;
2907 struct e1000_hw *hw = &adapter->hw;
2908 struct e1000_fc_info *fc = &hw->fc;
2909 struct ifnet *ifp = adapter->ifp;
2913 INIT_DEBUGOUT("igb_reset: begin");
2915 /* Let the firmware know the OS is in control */
2916 igb_get_hw_control(adapter);
2919 * Packet Buffer Allocation (PBA)
2920 * Writing PBA sets the receive portion of the buffer
2921 * the remainder is used for the transmit buffer.
2923 switch (hw->mac.type) {
2925 pba = E1000_PBA_32K;
2929 pba = E1000_READ_REG(hw, E1000_RXPBS);
2930 pba &= E1000_RXPBS_SIZE_MASK_82576;
2935 case e1000_vfadapt_i350:
2936 pba = E1000_READ_REG(hw, E1000_RXPBS);
2937 pba = e1000_rxpbs_adjust_82580(pba);
2941 pba = E1000_PBA_34K;
2946 /* Special needs in case of Jumbo frames */
2947 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
2948 u32 tx_space, min_tx, min_rx;
2949 pba = E1000_READ_REG(hw, E1000_PBA);
2950 tx_space = pba >> 16;
2952 min_tx = (adapter->max_frame_size +
2953 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
2954 min_tx = roundup2(min_tx, 1024);
2956 min_rx = adapter->max_frame_size;
2957 min_rx = roundup2(min_rx, 1024);
2959 if (tx_space < min_tx &&
2960 ((min_tx - tx_space) < pba)) {
2961 pba = pba - (min_tx - tx_space);
2963 * if short on rx space, rx wins
2964 * and must trump tx adjustment
2969 E1000_WRITE_REG(hw, E1000_PBA, pba);
2972 INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
2975 * These parameters control the automatic generation (Tx) and
2976 * response (Rx) to Ethernet PAUSE frames.
2977 * - High water mark should allow for at least two frames to be
2978 * received after sending an XOFF.
2979 * - Low water mark works best when it is very near the high water mark.
2980 * This allows the receiver to restart by sending XON when it has
2983 hwm = min(((pba << 10) * 9 / 10),
2984 ((pba << 10) - 2 * adapter->max_frame_size));
2986 if (hw->mac.type < e1000_82576) {
2987 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
2988 fc->low_water = fc->high_water - 8;
2990 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
2991 fc->low_water = fc->high_water - 16;
2994 fc->pause_time = IGB_FC_PAUSE_TIME;
2995 fc->send_xon = TRUE;
2997 fc->requested_mode = adapter->fc;
2999 fc->requested_mode = e1000_fc_default;
3001 /* Issue a global reset */
3003 E1000_WRITE_REG(hw, E1000_WUC, 0);
3005 /* Reset for AutoMediaDetect */
3006 if (adapter->flags & IGB_MEDIA_RESET) {
3007 e1000_setup_init_funcs(hw, TRUE);
3008 e1000_get_bus_info(hw);
3009 adapter->flags &= ~IGB_MEDIA_RESET;
3012 if (e1000_init_hw(hw) < 0)
3013 device_printf(dev, "Hardware Initialization Failed\n");
3015 /* Setup DMA Coalescing */
3016 igb_init_dmac(adapter, pba);
3018 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
3019 e1000_get_phy_info(hw);
3020 e1000_check_for_link(hw);
3024 /*********************************************************************
3026 * Setup networking device structure and register an interface.
3028 **********************************************************************/
3030 igb_setup_interface(device_t dev, struct adapter *adapter)
3034 INIT_DEBUGOUT("igb_setup_interface: begin");
3036 ifp = adapter->ifp = if_alloc(IFT_ETHER);
3038 device_printf(dev, "can not allocate ifnet structure\n");
3041 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3042 ifp->if_init = igb_init;
3043 ifp->if_softc = adapter;
3044 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3045 ifp->if_ioctl = igb_ioctl;
3047 /* TSO parameters */
3048 ifp->if_hw_tsomax = IP_MAXPACKET;
3049 ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER;
3050 ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE;
3052 #ifndef IGB_LEGACY_TX
3053 ifp->if_transmit = igb_mq_start;
3054 ifp->if_qflush = igb_qflush;
3056 ifp->if_start = igb_start;
3057 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3058 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3059 IFQ_SET_READY(&ifp->if_snd);
3062 ether_ifattach(ifp, adapter->hw.mac.addr);
3064 ifp->if_capabilities = ifp->if_capenable = 0;
3066 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3067 ifp->if_capabilities |= IFCAP_TSO;
3068 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
3069 ifp->if_capenable = ifp->if_capabilities;
3071 /* Don't enable LRO by default */
3072 ifp->if_capabilities |= IFCAP_LRO;
3074 #ifdef DEVICE_POLLING
3075 ifp->if_capabilities |= IFCAP_POLLING;
3079 * Tell the upper layer(s) we
3080 * support full VLAN capability.
3082 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3083 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
3086 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
3091 ** Don't turn this on by default, if vlans are
3092 ** created on another pseudo device (eg. lagg)
3093 ** then vlan events are not passed thru, breaking
3094 ** operation, but with HW FILTER off it works. If
3095 ** using vlans directly on the igb driver you can
3096 ** enable this and get full hardware tag filtering.
3098 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3101 * Specify the media types supported by this adapter and register
3102 * callbacks to update media and link information
3104 ifmedia_init(&adapter->media, IFM_IMASK,
3105 igb_media_change, igb_media_status);
3106 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3107 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3108 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
3110 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
3112 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3113 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3115 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3117 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3119 if (adapter->hw.phy.type != e1000_phy_ife) {
3120 ifmedia_add(&adapter->media,
3121 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3122 ifmedia_add(&adapter->media,
3123 IFM_ETHER | IFM_1000_T, 0, NULL);
3126 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3127 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3133 * Manage DMA'able memory.
3136 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3140 *(bus_addr_t *) arg = segs[0].ds_addr;
3144 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
3145 struct igb_dma_alloc *dma, int mapflags)
3149 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3150 IGB_DBA_ALIGN, 0, /* alignment, bounds */
3151 BUS_SPACE_MAXADDR, /* lowaddr */
3152 BUS_SPACE_MAXADDR, /* highaddr */
3153 NULL, NULL, /* filter, filterarg */
3156 size, /* maxsegsize */
3158 NULL, /* lockfunc */
3162 device_printf(adapter->dev,
3163 "%s: bus_dma_tag_create failed: %d\n",
3168 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3169 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3171 device_printf(adapter->dev,
3172 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3173 __func__, (uintmax_t)size, error);
3178 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3179 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3180 if (error || dma->dma_paddr == 0) {
3181 device_printf(adapter->dev,
3182 "%s: bus_dmamap_load failed: %d\n",
3190 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3192 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3193 bus_dma_tag_destroy(dma->dma_tag);
3195 dma->dma_tag = NULL;
3201 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
3203 if (dma->dma_tag == NULL)
3205 if (dma->dma_paddr != 0) {
3206 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3207 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3208 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3211 if (dma->dma_vaddr != NULL) {
3212 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3213 dma->dma_vaddr = NULL;
3215 bus_dma_tag_destroy(dma->dma_tag);
3216 dma->dma_tag = NULL;
3220 /*********************************************************************
3222 * Allocate memory for the transmit and receive rings, and then
3223 * the descriptors associated with each, called only once at attach.
3225 **********************************************************************/
3227 igb_allocate_queues(struct adapter *adapter)
3229 device_t dev = adapter->dev;
3230 struct igb_queue *que = NULL;
3231 struct tx_ring *txr = NULL;
3232 struct rx_ring *rxr = NULL;
3233 int rsize, tsize, error = E1000_SUCCESS;
3234 int txconf = 0, rxconf = 0;
3236 /* First allocate the top level queue structs */
3237 if (!(adapter->queues =
3238 (struct igb_queue *) malloc(sizeof(struct igb_queue) *
3239 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3240 device_printf(dev, "Unable to allocate queue memory\n");
3245 /* Next allocate the TX ring struct memory */
3246 if (!(adapter->tx_rings =
3247 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3248 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3249 device_printf(dev, "Unable to allocate TX ring memory\n");
3254 /* Now allocate the RX */
3255 if (!(adapter->rx_rings =
3256 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3257 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3258 device_printf(dev, "Unable to allocate RX ring memory\n");
3263 tsize = roundup2(adapter->num_tx_desc *
3264 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
3266 * Now set up the TX queues, txconf is needed to handle the
3267 * possibility that things fail midcourse and we need to
3268 * undo memory gracefully
3270 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3271 /* Set up some basics */
3272 txr = &adapter->tx_rings[i];
3273 txr->adapter = adapter;
3275 txr->num_desc = adapter->num_tx_desc;
3277 /* Initialize the TX lock */
3278 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3279 device_get_nameunit(dev), txr->me);
3280 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3282 if (igb_dma_malloc(adapter, tsize,
3283 &txr->txdma, BUS_DMA_NOWAIT)) {
3285 "Unable to allocate TX Descriptor memory\n");
3289 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
3290 bzero((void *)txr->tx_base, tsize);
3292 /* Now allocate transmit buffers for the ring */
3293 if (igb_allocate_transmit_buffers(txr)) {
3295 "Critical Failure setting up transmit buffers\n");
3299 #ifndef IGB_LEGACY_TX
3300 /* Allocate a buf ring */
3301 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
3302 M_WAITOK, &txr->tx_mtx);
3307 * Next the RX queues...
3309 rsize = roundup2(adapter->num_rx_desc *
3310 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3311 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3312 rxr = &adapter->rx_rings[i];
3313 rxr->adapter = adapter;
3316 /* Initialize the RX lock */
3317 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3318 device_get_nameunit(dev), txr->me);
3319 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3321 if (igb_dma_malloc(adapter, rsize,
3322 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3324 "Unable to allocate RxDescriptor memory\n");
3328 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3329 bzero((void *)rxr->rx_base, rsize);
3331 /* Allocate receive buffers for the ring*/
3332 if (igb_allocate_receive_buffers(rxr)) {
3334 "Critical Failure setting up receive buffers\n");
3341 ** Finally set up the queue holding structs
3343 for (int i = 0; i < adapter->num_queues; i++) {
3344 que = &adapter->queues[i];
3345 que->adapter = adapter;
3346 que->txr = &adapter->tx_rings[i];
3347 que->rxr = &adapter->rx_rings[i];
3353 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3354 igb_dma_free(adapter, &rxr->rxdma);
3356 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3357 igb_dma_free(adapter, &txr->txdma);
3358 free(adapter->rx_rings, M_DEVBUF);
3360 #ifndef IGB_LEGACY_TX
3361 buf_ring_free(txr->br, M_DEVBUF);
3363 free(adapter->tx_rings, M_DEVBUF);
3365 free(adapter->queues, M_DEVBUF);
3370 /*********************************************************************
3372 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3373 * the information needed to transmit a packet on the wire. This is
3374 * called only once at attach, setup is done every reset.
3376 **********************************************************************/
3378 igb_allocate_transmit_buffers(struct tx_ring *txr)
3380 struct adapter *adapter = txr->adapter;
3381 device_t dev = adapter->dev;
3382 struct igb_tx_buf *txbuf;
3386 * Setup DMA descriptor areas.
3388 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3389 1, 0, /* alignment, bounds */
3390 BUS_SPACE_MAXADDR, /* lowaddr */
3391 BUS_SPACE_MAXADDR, /* highaddr */
3392 NULL, NULL, /* filter, filterarg */
3393 IGB_TSO_SIZE, /* maxsize */
3394 IGB_MAX_SCATTER, /* nsegments */
3395 PAGE_SIZE, /* maxsegsize */
3397 NULL, /* lockfunc */
3398 NULL, /* lockfuncarg */
3400 device_printf(dev,"Unable to allocate TX DMA tag\n");
3404 if (!(txr->tx_buffers =
3405 (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
3406 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3407 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3412 /* Create the descriptor buffer dma maps */
3413 txbuf = txr->tx_buffers;
3414 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3415 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3417 device_printf(dev, "Unable to create TX DMA map\n");
3424 /* We free all, it handles case where we are in the middle */
3425 igb_free_transmit_structures(adapter);
3429 /*********************************************************************
3431 * Initialize a transmit ring.
3433 **********************************************************************/
3435 igb_setup_transmit_ring(struct tx_ring *txr)
3437 struct adapter *adapter = txr->adapter;
3438 struct igb_tx_buf *txbuf;
3441 struct netmap_adapter *na = NA(adapter->ifp);
3442 struct netmap_slot *slot;
3443 #endif /* DEV_NETMAP */
3445 /* Clear the old descriptor contents */
3448 slot = netmap_reset(na, NR_TX, txr->me, 0);
3449 #endif /* DEV_NETMAP */
3450 bzero((void *)txr->tx_base,
3451 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3453 txr->next_avail_desc = 0;
3454 txr->next_to_clean = 0;
3456 /* Free any existing tx buffers. */
3457 txbuf = txr->tx_buffers;
3458 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3459 if (txbuf->m_head != NULL) {
3460 bus_dmamap_sync(txr->txtag, txbuf->map,
3461 BUS_DMASYNC_POSTWRITE);
3462 bus_dmamap_unload(txr->txtag, txbuf->map);
3463 m_freem(txbuf->m_head);
3464 txbuf->m_head = NULL;
3468 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3469 /* no need to set the address */
3470 netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
3472 #endif /* DEV_NETMAP */
3473 /* clear the watch index */
3477 /* Set number of descriptors available */
3478 txr->tx_avail = adapter->num_tx_desc;
3480 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3481 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3485 /*********************************************************************
3487 * Initialize all transmit rings.
3489 **********************************************************************/
3491 igb_setup_transmit_structures(struct adapter *adapter)
3493 struct tx_ring *txr = adapter->tx_rings;
3495 for (int i = 0; i < adapter->num_queues; i++, txr++)
3496 igb_setup_transmit_ring(txr);
3501 /*********************************************************************
3503 * Enable transmit unit.
3505 **********************************************************************/
3507 igb_initialize_transmit_units(struct adapter *adapter)
3509 struct tx_ring *txr = adapter->tx_rings;
3510 struct e1000_hw *hw = &adapter->hw;
3513 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3516 /* Setup the Tx Descriptor Rings */
3517 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3518 u64 bus_addr = txr->txdma.dma_paddr;
3520 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3521 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3522 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3523 (uint32_t)(bus_addr >> 32));
3524 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3525 (uint32_t)bus_addr);
3527 /* Setup the HW Tx Head and Tail descriptor pointers */
3528 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3529 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3531 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3532 E1000_READ_REG(hw, E1000_TDBAL(i)),
3533 E1000_READ_REG(hw, E1000_TDLEN(i)));
3535 txr->queue_status = IGB_QUEUE_IDLE;
3537 txdctl |= IGB_TX_PTHRESH;
3538 txdctl |= IGB_TX_HTHRESH << 8;
3539 txdctl |= IGB_TX_WTHRESH << 16;
3540 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3541 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3544 if (adapter->vf_ifp)
3547 e1000_config_collision_dist(hw);
3549 /* Program the Transmit Control Register */
3550 tctl = E1000_READ_REG(hw, E1000_TCTL);
3551 tctl &= ~E1000_TCTL_CT;
3552 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3553 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3555 /* This write will effectively turn on the transmit unit. */
3556 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3559 /*********************************************************************
3561 * Free all transmit rings.
3563 **********************************************************************/
3565 igb_free_transmit_structures(struct adapter *adapter)
3567 struct tx_ring *txr = adapter->tx_rings;
3569 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3571 igb_free_transmit_buffers(txr);
3572 igb_dma_free(adapter, &txr->txdma);
3574 IGB_TX_LOCK_DESTROY(txr);
3576 free(adapter->tx_rings, M_DEVBUF);
3579 /*********************************************************************
3581 * Free transmit ring related data structures.
3583 **********************************************************************/
3585 igb_free_transmit_buffers(struct tx_ring *txr)
3587 struct adapter *adapter = txr->adapter;
3588 struct igb_tx_buf *tx_buffer;
3591 INIT_DEBUGOUT("free_transmit_ring: begin");
3593 if (txr->tx_buffers == NULL)
3596 tx_buffer = txr->tx_buffers;
3597 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3598 if (tx_buffer->m_head != NULL) {
3599 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3600 BUS_DMASYNC_POSTWRITE);
3601 bus_dmamap_unload(txr->txtag,
3603 m_freem(tx_buffer->m_head);
3604 tx_buffer->m_head = NULL;
3605 if (tx_buffer->map != NULL) {
3606 bus_dmamap_destroy(txr->txtag,
3608 tx_buffer->map = NULL;
3610 } else if (tx_buffer->map != NULL) {
3611 bus_dmamap_unload(txr->txtag,
3613 bus_dmamap_destroy(txr->txtag,
3615 tx_buffer->map = NULL;
3618 #ifndef IGB_LEGACY_TX
3619 if (txr->br != NULL)
3620 buf_ring_free(txr->br, M_DEVBUF);
3622 if (txr->tx_buffers != NULL) {
3623 free(txr->tx_buffers, M_DEVBUF);
3624 txr->tx_buffers = NULL;
3626 if (txr->txtag != NULL) {
3627 bus_dma_tag_destroy(txr->txtag);
3633 /**********************************************************************
3635 * Setup work for hardware segmentation offload (TSO) on
3636 * adapters using advanced tx descriptors
3638 **********************************************************************/
3640 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3641 u32 *cmd_type_len, u32 *olinfo_status)
3643 struct adapter *adapter = txr->adapter;
3644 struct e1000_adv_tx_context_desc *TXD;
3645 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3646 u32 mss_l4len_idx = 0, paylen;
3647 u16 vtag = 0, eh_type;
3648 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3649 struct ether_vlan_header *eh;
3651 struct ip6_hdr *ip6;
3660 * Determine where frame payload starts.
3661 * Jump over vlan headers if already present
3663 eh = mtod(mp, struct ether_vlan_header *);
3664 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3665 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3666 eh_type = eh->evl_proto;
3668 ehdrlen = ETHER_HDR_LEN;
3669 eh_type = eh->evl_encap_proto;
3672 switch (ntohs(eh_type)) {
3674 case ETHERTYPE_IPV6:
3675 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3676 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3677 if (ip6->ip6_nxt != IPPROTO_TCP)
3679 ip_hlen = sizeof(struct ip6_hdr);
3680 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3681 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3682 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3683 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3688 ip = (struct ip *)(mp->m_data + ehdrlen);
3689 if (ip->ip_p != IPPROTO_TCP)
3692 ip_hlen = ip->ip_hl << 2;
3693 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3694 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3695 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3696 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3697 /* Tell transmit desc to also do IPv4 checksum. */
3698 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3702 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3703 __func__, ntohs(eh_type));
3707 ctxd = txr->next_avail_desc;
3708 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3710 tcp_hlen = th->th_off << 2;
3712 /* This is used in the transmit desc in encap */
3713 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3715 /* VLAN MACLEN IPLEN */
3716 if (mp->m_flags & M_VLANTAG) {
3717 vtag = htole16(mp->m_pkthdr.ether_vtag);
3718 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3721 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3722 vlan_macip_lens |= ip_hlen;
3723 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3725 /* ADV DTYPE TUCMD */
3726 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3727 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3728 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3731 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3732 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3733 /* 82575 needs the queue index added */
3734 if (adapter->hw.mac.type == e1000_82575)
3735 mss_l4len_idx |= txr->me << 4;
3736 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3738 TXD->seqnum_seed = htole32(0);
3740 if (++ctxd == txr->num_desc)
3744 txr->next_avail_desc = ctxd;
3745 *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3746 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3747 *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
3752 /*********************************************************************
3754 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3756 **********************************************************************/
3759 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3760 u32 *cmd_type_len, u32 *olinfo_status)
3762 struct e1000_adv_tx_context_desc *TXD;
3763 struct adapter *adapter = txr->adapter;
3764 struct ether_vlan_header *eh;
3766 struct ip6_hdr *ip6;
3767 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
3768 int ehdrlen, ip_hlen = 0;
3772 int ctxd = txr->next_avail_desc;
3775 /* First check if TSO is to be used */
3776 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3777 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3779 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3782 /* Indicate the whole packet as payload when not doing TSO */
3783 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
3785 /* Now ready a context descriptor */
3786 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3789 ** In advanced descriptors the vlan tag must
3790 ** be placed into the context descriptor. Hence
3791 ** we need to make one even if not doing offloads.
3793 if (mp->m_flags & M_VLANTAG) {
3794 vtag = htole16(mp->m_pkthdr.ether_vtag);
3795 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3796 } else if (offload == FALSE) /* ... no offload to do */
3800 * Determine where frame payload starts.
3801 * Jump over vlan headers if already present,
3802 * helpful for QinQ too.
3804 eh = mtod(mp, struct ether_vlan_header *);
3805 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3806 etype = ntohs(eh->evl_proto);
3807 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3809 etype = ntohs(eh->evl_encap_proto);
3810 ehdrlen = ETHER_HDR_LEN;
3813 /* Set the ether header length */
3814 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3818 ip = (struct ip *)(mp->m_data + ehdrlen);
3819 ip_hlen = ip->ip_hl << 2;
3821 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3823 case ETHERTYPE_IPV6:
3824 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3825 ip_hlen = sizeof(struct ip6_hdr);
3826 /* XXX-BZ this will go badly in case of ext hdrs. */
3827 ipproto = ip6->ip6_nxt;
3828 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3835 vlan_macip_lens |= ip_hlen;
3836 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3840 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3841 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3844 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3845 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3848 #if __FreeBSD_version >= 800000
3850 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3851 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3859 if (offload) /* For the TX descriptor setup */
3860 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3862 /* 82575 needs the queue index added */
3863 if (adapter->hw.mac.type == e1000_82575)
3864 mss_l4len_idx = txr->me << 4;
3866 /* Now copy bits into descriptor */
3867 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3868 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3869 TXD->seqnum_seed = htole32(0);
3870 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3872 /* We've consumed the first desc, adjust counters */
3873 if (++ctxd == txr->num_desc)
3875 txr->next_avail_desc = ctxd;
3881 /**********************************************************************
3883 * Examine each tx_buffer in the used queue. If the hardware is done
3884 * processing the packet then free associated resources. The
3885 * tx_buffer is put back on the free queue.
3887 * TRUE return means there's work in the ring to clean, FALSE its empty.
3888 **********************************************************************/
3890 igb_txeof(struct tx_ring *txr)
3892 struct adapter *adapter = txr->adapter;
3893 struct ifnet *ifp = adapter->ifp;
3894 u32 work, processed = 0;
3895 int limit = adapter->tx_process_limit;
3896 struct igb_tx_buf *buf;
3897 union e1000_adv_tx_desc *txd;
3899 mtx_assert(&txr->tx_mtx, MA_OWNED);
3902 if (netmap_tx_irq(ifp, txr->me))
3904 #endif /* DEV_NETMAP */
3906 if (txr->tx_avail == txr->num_desc) {
3907 txr->queue_status = IGB_QUEUE_IDLE;
3911 /* Get work starting point */
3912 work = txr->next_to_clean;
3913 buf = &txr->tx_buffers[work];
3914 txd = &txr->tx_base[work];
3915 work -= txr->num_desc; /* The distance to ring end */
3916 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3917 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3919 union e1000_adv_tx_desc *eop = buf->eop;
3920 if (eop == NULL) /* No work */
3923 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
3924 break; /* I/O not complete */
3928 buf->m_head->m_pkthdr.len;
3929 bus_dmamap_sync(txr->txtag,
3931 BUS_DMASYNC_POSTWRITE);
3932 bus_dmamap_unload(txr->txtag,
3934 m_freem(buf->m_head);
3940 /* We clean the range if multi segment */
3941 while (txd != eop) {
3945 /* wrap the ring? */
3946 if (__predict_false(!work)) {
3947 work -= txr->num_desc;
3948 buf = txr->tx_buffers;
3953 buf->m_head->m_pkthdr.len;
3954 bus_dmamap_sync(txr->txtag,
3956 BUS_DMASYNC_POSTWRITE);
3957 bus_dmamap_unload(txr->txtag,
3959 m_freem(buf->m_head);
3969 txr->watchdog_time = ticks;
3971 /* Try the next packet */
3975 /* reset with a wrap */
3976 if (__predict_false(!work)) {
3977 work -= txr->num_desc;
3978 buf = txr->tx_buffers;
3982 } while (__predict_true(--limit));
3984 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3985 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3987 work += txr->num_desc;
3988 txr->next_to_clean = work;
3991 ** Watchdog calculation, we know there's
3992 ** work outstanding or the first return
3993 ** would have been taken, so none processed
3994 ** for too long indicates a hang.
3996 if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
3997 txr->queue_status |= IGB_QUEUE_HUNG;
3999 if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
4000 txr->queue_status &= ~IGB_QUEUE_DEPLETED;
4002 if (txr->tx_avail == txr->num_desc) {
4003 txr->queue_status = IGB_QUEUE_IDLE;
4010 /*********************************************************************
4012 * Refresh mbuf buffers for RX descriptor rings
4013 * - now keeps its own state so discards due to resource
4014 * exhaustion are unnecessary, if an mbuf cannot be obtained
4015 * it just returns, keeping its placeholder, thus it can simply
4016 * be recalled to try again.
4018 **********************************************************************/
4020 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
4022 struct adapter *adapter = rxr->adapter;
4023 bus_dma_segment_t hseg[1];
4024 bus_dma_segment_t pseg[1];
4025 struct igb_rx_buf *rxbuf;
4026 struct mbuf *mh, *mp;
4027 int i, j, nsegs, error;
4028 bool refreshed = FALSE;
4030 i = j = rxr->next_to_refresh;
4032 ** Get one descriptor beyond
4033 ** our work mark to control
4036 if (++j == adapter->num_rx_desc)
4039 while (j != limit) {
4040 rxbuf = &rxr->rx_buffers[i];
4041 /* No hdr mbuf used with header split off */
4042 if (rxr->hdr_split == FALSE)
4044 if (rxbuf->m_head == NULL) {
4045 mh = m_gethdr(M_NOWAIT, MT_DATA);
4051 mh->m_pkthdr.len = mh->m_len = MHLEN;
4053 mh->m_flags |= M_PKTHDR;
4054 /* Get the memory mapping */
4055 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4056 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
4058 printf("Refresh mbufs: hdr dmamap load"
4059 " failure - %d\n", error);
4061 rxbuf->m_head = NULL;
4065 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4066 BUS_DMASYNC_PREREAD);
4067 rxr->rx_base[i].read.hdr_addr =
4068 htole64(hseg[0].ds_addr);
4070 if (rxbuf->m_pack == NULL) {
4071 mp = m_getjcl(M_NOWAIT, MT_DATA,
4072 M_PKTHDR, adapter->rx_mbuf_sz);
4078 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4079 /* Get the memory mapping */
4080 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4081 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
4083 printf("Refresh mbufs: payload dmamap load"
4084 " failure - %d\n", error);
4086 rxbuf->m_pack = NULL;
4090 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4091 BUS_DMASYNC_PREREAD);
4092 rxr->rx_base[i].read.pkt_addr =
4093 htole64(pseg[0].ds_addr);
4094 refreshed = TRUE; /* I feel wefreshed :) */
4096 i = j; /* our next is precalculated */
4097 rxr->next_to_refresh = i;
4098 if (++j == adapter->num_rx_desc)
4102 if (refreshed) /* update tail */
4103 E1000_WRITE_REG(&adapter->hw,
4104 E1000_RDT(rxr->me), rxr->next_to_refresh);
4109 /*********************************************************************
4111 * Allocate memory for rx_buffer structures. Since we use one
4112 * rx_buffer per received packet, the maximum number of rx_buffer's
4113 * that we'll need is equal to the number of receive descriptors
4114 * that we've allocated.
4116 **********************************************************************/
4118 igb_allocate_receive_buffers(struct rx_ring *rxr)
4120 struct adapter *adapter = rxr->adapter;
4121 device_t dev = adapter->dev;
4122 struct igb_rx_buf *rxbuf;
4123 int i, bsize, error;
4125 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
4126 if (!(rxr->rx_buffers =
4127 (struct igb_rx_buf *) malloc(bsize,
4128 M_DEVBUF, M_NOWAIT | M_ZERO))) {
4129 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4134 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4135 1, 0, /* alignment, bounds */
4136 BUS_SPACE_MAXADDR, /* lowaddr */
4137 BUS_SPACE_MAXADDR, /* highaddr */
4138 NULL, NULL, /* filter, filterarg */
4139 MSIZE, /* maxsize */
4141 MSIZE, /* maxsegsize */
4143 NULL, /* lockfunc */
4144 NULL, /* lockfuncarg */
4146 device_printf(dev, "Unable to create RX DMA tag\n");
4150 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4151 1, 0, /* alignment, bounds */
4152 BUS_SPACE_MAXADDR, /* lowaddr */
4153 BUS_SPACE_MAXADDR, /* highaddr */
4154 NULL, NULL, /* filter, filterarg */
4155 MJUM9BYTES, /* maxsize */
4157 MJUM9BYTES, /* maxsegsize */
4159 NULL, /* lockfunc */
4160 NULL, /* lockfuncarg */
4162 device_printf(dev, "Unable to create RX payload DMA tag\n");
4166 for (i = 0; i < adapter->num_rx_desc; i++) {
4167 rxbuf = &rxr->rx_buffers[i];
4168 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
4171 "Unable to create RX head DMA maps\n");
4174 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
4177 "Unable to create RX packet DMA maps\n");
4185 /* Frees all, but can handle partial completion */
4186 igb_free_receive_structures(adapter);
4192 igb_free_receive_ring(struct rx_ring *rxr)
4194 struct adapter *adapter = rxr->adapter;
4195 struct igb_rx_buf *rxbuf;
4198 for (int i = 0; i < adapter->num_rx_desc; i++) {
4199 rxbuf = &rxr->rx_buffers[i];
4200 if (rxbuf->m_head != NULL) {
4201 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4202 BUS_DMASYNC_POSTREAD);
4203 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4204 rxbuf->m_head->m_flags |= M_PKTHDR;
4205 m_freem(rxbuf->m_head);
4207 if (rxbuf->m_pack != NULL) {
4208 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4209 BUS_DMASYNC_POSTREAD);
4210 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4211 rxbuf->m_pack->m_flags |= M_PKTHDR;
4212 m_freem(rxbuf->m_pack);
4214 rxbuf->m_head = NULL;
4215 rxbuf->m_pack = NULL;
4220 /*********************************************************************
4222 * Initialize a receive ring and its buffers.
4224 **********************************************************************/
4226 igb_setup_receive_ring(struct rx_ring *rxr)
4228 struct adapter *adapter;
4231 struct igb_rx_buf *rxbuf;
4232 bus_dma_segment_t pseg[1], hseg[1];
4233 struct lro_ctrl *lro = &rxr->lro;
4234 int rsize, nsegs, error = 0;
4236 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4237 struct netmap_slot *slot;
4238 #endif /* DEV_NETMAP */
4240 adapter = rxr->adapter;
4244 /* Clear the ring contents */
4247 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4248 #endif /* DEV_NETMAP */
4249 rsize = roundup2(adapter->num_rx_desc *
4250 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
4251 bzero((void *)rxr->rx_base, rsize);
4254 ** Free current RX buffer structures and their mbufs
4256 igb_free_receive_ring(rxr);
4258 /* Configure for header split? */
4259 if (igb_header_split)
4260 rxr->hdr_split = TRUE;
4262 /* Now replenish the ring mbufs */
4263 for (int j = 0; j < adapter->num_rx_desc; ++j) {
4264 struct mbuf *mh, *mp;
4266 rxbuf = &rxr->rx_buffers[j];
4269 /* slot sj is mapped to the i-th NIC-ring entry */
4270 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4274 addr = PNMB(na, slot + sj, &paddr);
4275 netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
4276 /* Update descriptor */
4277 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4280 #endif /* DEV_NETMAP */
4281 if (rxr->hdr_split == FALSE)
4284 /* First the header */
4285 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4286 if (rxbuf->m_head == NULL) {
4290 m_adj(rxbuf->m_head, ETHER_ALIGN);
4292 mh->m_len = mh->m_pkthdr.len = MHLEN;
4293 mh->m_flags |= M_PKTHDR;
4294 /* Get the memory mapping */
4295 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4296 rxbuf->hmap, rxbuf->m_head, hseg,
4297 &nsegs, BUS_DMA_NOWAIT);
4298 if (error != 0) /* Nothing elegant to do here */
4300 bus_dmamap_sync(rxr->htag,
4301 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4302 /* Update descriptor */
4303 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4306 /* Now the payload cluster */
4307 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4308 M_PKTHDR, adapter->rx_mbuf_sz);
4309 if (rxbuf->m_pack == NULL) {
4314 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4315 /* Get the memory mapping */
4316 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4317 rxbuf->pmap, mp, pseg,
4318 &nsegs, BUS_DMA_NOWAIT);
4321 bus_dmamap_sync(rxr->ptag,
4322 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4323 /* Update descriptor */
4324 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4327 /* Setup our descriptor indices */
4328 rxr->next_to_check = 0;
4329 rxr->next_to_refresh = adapter->num_rx_desc - 1;
4330 rxr->lro_enabled = FALSE;
4331 rxr->rx_split_packets = 0;
4337 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4338 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4341 ** Now set up the LRO interface, we
4342 ** also only do head split when LRO
4343 ** is enabled, since so often they
4344 ** are undesireable in similar setups.
4346 if (ifp->if_capenable & IFCAP_LRO) {
4347 error = tcp_lro_init(lro);
4349 device_printf(dev, "LRO Initialization failed!\n");
4352 INIT_DEBUGOUT("RX LRO Initialized\n");
4353 rxr->lro_enabled = TRUE;
4354 lro->ifp = adapter->ifp;
4361 igb_free_receive_ring(rxr);
4367 /*********************************************************************
4369 * Initialize all receive rings.
4371 **********************************************************************/
4373 igb_setup_receive_structures(struct adapter *adapter)
4375 struct rx_ring *rxr = adapter->rx_rings;
4378 for (i = 0; i < adapter->num_queues; i++, rxr++)
4379 if (igb_setup_receive_ring(rxr))
4385 * Free RX buffers allocated so far, we will only handle
4386 * the rings that completed, the failing case will have
4387 * cleaned up for itself. 'i' is the endpoint.
4389 for (int j = 0; j < i; ++j) {
4390 rxr = &adapter->rx_rings[j];
4392 igb_free_receive_ring(rxr);
4399 /*********************************************************************
4401 * Enable receive unit.
4403 **********************************************************************/
4405 igb_initialize_receive_units(struct adapter *adapter)
4407 struct rx_ring *rxr = adapter->rx_rings;
4408 struct ifnet *ifp = adapter->ifp;
4409 struct e1000_hw *hw = &adapter->hw;
4410 u32 rctl, rxcsum, psize, srrctl = 0;
4412 INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
4415 * Make sure receives are disabled while setting
4416 * up the descriptor ring
4418 rctl = E1000_READ_REG(hw, E1000_RCTL);
4419 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4422 ** Set up for header split
4424 if (igb_header_split) {
4425 /* Use a standard mbuf for the header */
4426 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4427 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4429 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4432 ** Set up for jumbo frames
4434 if (ifp->if_mtu > ETHERMTU) {
4435 rctl |= E1000_RCTL_LPE;
4436 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
4437 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4438 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4439 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
4440 srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4441 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4443 /* Set maximum packet len */
4444 psize = adapter->max_frame_size;
4445 /* are we on a vlan? */
4446 if (adapter->ifp->if_vlantrunk != NULL)
4447 psize += VLAN_TAG_SIZE;
4448 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
4450 rctl &= ~E1000_RCTL_LPE;
4451 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4452 rctl |= E1000_RCTL_SZ_2048;
4456 * If TX flow control is disabled and there's >1 queue defined,
4459 * This drops frames rather than hanging the RX MAC for all queues.
4461 if ((adapter->num_queues > 1) &&
4462 (adapter->fc == e1000_fc_none ||
4463 adapter->fc == e1000_fc_rx_pause)) {
4464 srrctl |= E1000_SRRCTL_DROP_EN;
4467 /* Setup the Base and Length of the Rx Descriptor Rings */
4468 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4469 u64 bus_addr = rxr->rxdma.dma_paddr;
4472 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4473 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4474 E1000_WRITE_REG(hw, E1000_RDBAH(i),
4475 (uint32_t)(bus_addr >> 32));
4476 E1000_WRITE_REG(hw, E1000_RDBAL(i),
4477 (uint32_t)bus_addr);
4478 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
4479 /* Enable this Queue */
4480 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
4481 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4482 rxdctl &= 0xFFF00000;
4483 rxdctl |= IGB_RX_PTHRESH;
4484 rxdctl |= IGB_RX_HTHRESH << 8;
4485 rxdctl |= IGB_RX_WTHRESH << 16;
4486 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
4490 ** Setup for RX MultiQueue
4492 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4493 if (adapter->num_queues >1) {
4494 u32 random[10], mrqc, shift = 0;
4500 arc4rand(&random, sizeof(random), 0);
4501 if (adapter->hw.mac.type == e1000_82575)
4503 /* Warning FM follows */
4504 for (int i = 0; i < 128; i++) {
4506 (i % adapter->num_queues) << shift;
4509 E1000_RETA(i >> 2), reta.dword);
4511 /* Now fill in hash table */
4512 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
4513 for (int i = 0; i < 10; i++)
4514 E1000_WRITE_REG_ARRAY(hw,
4515 E1000_RSSRK(0), i, random[i]);
4517 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
4518 E1000_MRQC_RSS_FIELD_IPV4_TCP);
4519 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
4520 E1000_MRQC_RSS_FIELD_IPV6_TCP);
4521 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
4522 E1000_MRQC_RSS_FIELD_IPV6_UDP);
4523 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
4524 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
4526 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
4529 ** NOTE: Receive Full-Packet Checksum Offload
4530 ** is mutually exclusive with Multiqueue. However
4531 ** this is not the same as TCP/IP checksums which
4534 rxcsum |= E1000_RXCSUM_PCSD;
4535 #if __FreeBSD_version >= 800000
4536 /* For SCTP Offload */
4537 if (((hw->mac.type == e1000_82576) ||
4538 (hw->mac.type == e1000_82580)) &&
4539 (ifp->if_capenable & IFCAP_RXCSUM))
4540 rxcsum |= E1000_RXCSUM_CRCOFL;
4544 if (ifp->if_capenable & IFCAP_RXCSUM) {
4545 rxcsum |= E1000_RXCSUM_IPPCSE;
4546 #if __FreeBSD_version >= 800000
4547 if ((adapter->hw.mac.type == e1000_82576) ||
4548 (adapter->hw.mac.type == e1000_82580))
4549 rxcsum |= E1000_RXCSUM_CRCOFL;
4552 rxcsum &= ~E1000_RXCSUM_TUOFL;
4554 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4556 /* Setup the Receive Control Register */
4557 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4558 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4559 E1000_RCTL_RDMTS_HALF |
4560 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4561 /* Strip CRC bytes. */
4562 rctl |= E1000_RCTL_SECRC;
4563 /* Make sure VLAN Filters are off */
4564 rctl &= ~E1000_RCTL_VFE;
4565 /* Don't store bad packets */
4566 rctl &= ~E1000_RCTL_SBP;
4568 /* Enable Receives */
4569 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4572 * Setup the HW Rx Head and Tail Descriptor Pointers
4573 * - needs to be after enable
4575 for (int i = 0; i < adapter->num_queues; i++) {
4576 rxr = &adapter->rx_rings[i];
4577 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
4580 * an init() while a netmap client is active must
4581 * preserve the rx buffers passed to userspace.
4582 * In this driver it means we adjust RDT to
4583 * something different from next_to_refresh
4584 * (which is not used in netmap mode).
4586 if (ifp->if_capenable & IFCAP_NETMAP) {
4587 struct netmap_adapter *na = NA(adapter->ifp);
4588 struct netmap_kring *kring = &na->rx_rings[i];
4589 int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
4591 if (t >= adapter->num_rx_desc)
4592 t -= adapter->num_rx_desc;
4594 t += adapter->num_rx_desc;
4595 E1000_WRITE_REG(hw, E1000_RDT(i), t);
4597 #endif /* DEV_NETMAP */
4598 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
4603 /*********************************************************************
4605 * Free receive rings.
4607 **********************************************************************/
4609 igb_free_receive_structures(struct adapter *adapter)
4611 struct rx_ring *rxr = adapter->rx_rings;
4613 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4614 struct lro_ctrl *lro = &rxr->lro;
4615 igb_free_receive_buffers(rxr);
4617 igb_dma_free(adapter, &rxr->rxdma);
4620 free(adapter->rx_rings, M_DEVBUF);
4623 /*********************************************************************
4625 * Free receive ring data structures.
4627 **********************************************************************/
4629 igb_free_receive_buffers(struct rx_ring *rxr)
4631 struct adapter *adapter = rxr->adapter;
4632 struct igb_rx_buf *rxbuf;
4635 INIT_DEBUGOUT("free_receive_structures: begin");
4637 /* Cleanup any existing buffers */
4638 if (rxr->rx_buffers != NULL) {
4639 for (i = 0; i < adapter->num_rx_desc; i++) {
4640 rxbuf = &rxr->rx_buffers[i];
4641 if (rxbuf->m_head != NULL) {
4642 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4643 BUS_DMASYNC_POSTREAD);
4644 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4645 rxbuf->m_head->m_flags |= M_PKTHDR;
4646 m_freem(rxbuf->m_head);
4648 if (rxbuf->m_pack != NULL) {
4649 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4650 BUS_DMASYNC_POSTREAD);
4651 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4652 rxbuf->m_pack->m_flags |= M_PKTHDR;
4653 m_freem(rxbuf->m_pack);
4655 rxbuf->m_head = NULL;
4656 rxbuf->m_pack = NULL;
4657 if (rxbuf->hmap != NULL) {
4658 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4661 if (rxbuf->pmap != NULL) {
4662 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4666 if (rxr->rx_buffers != NULL) {
4667 free(rxr->rx_buffers, M_DEVBUF);
4668 rxr->rx_buffers = NULL;
4672 if (rxr->htag != NULL) {
4673 bus_dma_tag_destroy(rxr->htag);
4676 if (rxr->ptag != NULL) {
4677 bus_dma_tag_destroy(rxr->ptag);
4682 static __inline void
4683 igb_rx_discard(struct rx_ring *rxr, int i)
4685 struct igb_rx_buf *rbuf;
4687 rbuf = &rxr->rx_buffers[i];
4689 /* Partially received? Free the chain */
4690 if (rxr->fmp != NULL) {
4691 rxr->fmp->m_flags |= M_PKTHDR;
4698 ** With advanced descriptors the writeback
4699 ** clobbers the buffer addrs, so its easier
4700 ** to just free the existing mbufs and take
4701 ** the normal refresh path to get new buffers
4705 m_free(rbuf->m_head);
4706 rbuf->m_head = NULL;
4707 bus_dmamap_unload(rxr->htag, rbuf->hmap);
4711 m_free(rbuf->m_pack);
4712 rbuf->m_pack = NULL;
4713 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
4719 static __inline void
4720 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4724 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4725 * should be computed by hardware. Also it should not have VLAN tag in
4728 if (rxr->lro_enabled &&
4729 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4730 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4731 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4732 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4733 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4734 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4736 * Send to the stack if:
4737 ** - LRO not enabled, or
4738 ** - no LRO resources, or
4739 ** - lro enqueue fails
4741 if (rxr->lro.lro_cnt != 0)
4742 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4746 (*ifp->if_input)(ifp, m);
4750 /*********************************************************************
4752 * This routine executes in interrupt context. It replenishes
4753 * the mbufs in the descriptor and sends data which has been
4754 * dma'ed into host memory to upper layer.
4756 * We loop at most count times if count is > 0, or until done if
4759 * Return TRUE if more to clean, FALSE otherwise
4760 *********************************************************************/
4762 igb_rxeof(struct igb_queue *que, int count, int *done)
4764 struct adapter *adapter = que->adapter;
4765 struct rx_ring *rxr = que->rxr;
4766 struct ifnet *ifp = adapter->ifp;
4767 struct lro_ctrl *lro = &rxr->lro;
4768 struct lro_entry *queued;
4769 int i, processed = 0, rxdone = 0;
4770 u32 ptype, staterr = 0;
4771 union e1000_adv_rx_desc *cur;
4774 /* Sync the ring. */
4775 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4776 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4779 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4783 #endif /* DEV_NETMAP */
4785 /* Main clean loop */
4786 for (i = rxr->next_to_check; count != 0;) {
4787 struct mbuf *sendmp, *mh, *mp;
4788 struct igb_rx_buf *rxbuf;
4789 u16 hlen, plen, hdr, vtag;
4792 cur = &rxr->rx_base[i];
4793 staterr = le32toh(cur->wb.upper.status_error);
4794 if ((staterr & E1000_RXD_STAT_DD) == 0)
4796 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4799 sendmp = mh = mp = NULL;
4800 cur->wb.upper.status_error = 0;
4801 rxbuf = &rxr->rx_buffers[i];
4802 plen = le16toh(cur->wb.upper.length);
4803 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
4804 if (((adapter->hw.mac.type == e1000_i350) ||
4805 (adapter->hw.mac.type == e1000_i354)) &&
4806 (staterr & E1000_RXDEXT_STATERR_LB))
4807 vtag = be16toh(cur->wb.upper.vlan);
4809 vtag = le16toh(cur->wb.upper.vlan);
4810 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4811 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
4814 * Free the frame (all segments) if we're at EOP and
4817 * The datasheet states that EOP + status is only valid for
4818 * the final segment in a multi-segment frame.
4820 if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
4821 adapter->dropped_pkts++;
4822 ++rxr->rx_discarded;
4823 igb_rx_discard(rxr, i);
4828 ** The way the hardware is configured to
4829 ** split, it will ONLY use the header buffer
4830 ** when header split is enabled, otherwise we
4831 ** get normal behavior, ie, both header and
4832 ** payload are DMA'd into the payload buffer.
4834 ** The fmp test is to catch the case where a
4835 ** packet spans multiple descriptors, in that
4836 ** case only the first header is valid.
4838 if (rxr->hdr_split && rxr->fmp == NULL) {
4839 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4840 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4841 E1000_RXDADV_HDRBUFLEN_SHIFT;
4842 if (hlen > IGB_HDR_BUF)
4844 mh = rxr->rx_buffers[i].m_head;
4846 /* clear buf pointer for refresh */
4847 rxbuf->m_head = NULL;
4849 ** Get the payload length, this
4850 ** could be zero if its a small
4854 mp = rxr->rx_buffers[i].m_pack;
4857 /* clear buf pointer */
4858 rxbuf->m_pack = NULL;
4859 rxr->rx_split_packets++;
4863 ** Either no header split, or a
4864 ** secondary piece of a fragmented
4867 mh = rxr->rx_buffers[i].m_pack;
4869 /* clear buf info for refresh */
4870 rxbuf->m_pack = NULL;
4872 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4874 ++processed; /* So we know when to refresh */
4876 /* Initial frame - setup */
4877 if (rxr->fmp == NULL) {
4878 mh->m_pkthdr.len = mh->m_len;
4879 /* Save the head of the chain */
4883 /* Add payload if split */
4884 mh->m_pkthdr.len += mp->m_len;
4885 rxr->lmp = mh->m_next;
4888 /* Chain mbuf's together */
4889 rxr->lmp->m_next = mh;
4890 rxr->lmp = rxr->lmp->m_next;
4891 rxr->fmp->m_pkthdr.len += mh->m_len;
4895 rxr->fmp->m_pkthdr.rcvif = ifp;
4898 /* capture data for AIM */
4900 rxr->bytes += rxr->fmp->m_pkthdr.len;
4901 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4903 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4904 igb_rx_checksum(staterr, rxr->fmp, ptype);
4906 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4907 (staterr & E1000_RXD_STAT_VP) != 0) {
4908 rxr->fmp->m_pkthdr.ether_vtag = vtag;
4909 rxr->fmp->m_flags |= M_VLANTAG;
4913 * In case of multiqueue, we have RXCSUM.PCSD bit set
4914 * and never cleared. This means we have RSS hash
4915 * available to be used.
4917 if (adapter->num_queues > 1) {
4918 rxr->fmp->m_pkthdr.flowid =
4919 le32toh(cur->wb.lower.hi_dword.rss);
4921 * Full RSS support is not avilable in
4922 * FreeBSD 10 so setting the hash type to
4925 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
4927 #ifndef IGB_LEGACY_TX
4928 rxr->fmp->m_pkthdr.flowid = que->msix;
4929 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
4933 /* Make sure to set M_PKTHDR. */
4934 sendmp->m_flags |= M_PKTHDR;
4940 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4941 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4943 /* Advance our pointers to the next descriptor. */
4944 if (++i == adapter->num_rx_desc)
4947 ** Send to the stack or LRO
4949 if (sendmp != NULL) {
4950 rxr->next_to_check = i;
4951 igb_rx_input(rxr, ifp, sendmp, ptype);
4952 i = rxr->next_to_check;
4956 /* Every 8 descriptors we go to refresh mbufs */
4957 if (processed == 8) {
4958 igb_refresh_mbufs(rxr, i);
4963 /* Catch any remainders */
4964 if (igb_rx_unrefreshed(rxr))
4965 igb_refresh_mbufs(rxr, i);
4967 rxr->next_to_check = i;
4970 * Flush any outstanding LRO work
4972 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4973 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4974 tcp_lro_flush(lro, queued);
4981 return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4984 /*********************************************************************
4986 * Verify that the hardware indicated that the checksum is valid.
4987 * Inform the stack about the status of checksum so that stack
4988 * doesn't spend time verifying the checksum.
4990 *********************************************************************/
4992 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
4994 u16 status = (u16)staterr;
4995 u8 errors = (u8) (staterr >> 24);
4998 /* Ignore Checksum bit is set */
4999 if (status & E1000_RXD_STAT_IXSM) {
5000 mp->m_pkthdr.csum_flags = 0;
5004 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
5005 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
5009 if (status & E1000_RXD_STAT_IPCS) {
5011 if (!(errors & E1000_RXD_ERR_IPE)) {
5012 /* IP Checksum Good */
5013 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
5014 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
5016 mp->m_pkthdr.csum_flags = 0;
5019 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
5020 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5021 #if __FreeBSD_version >= 800000
5022 if (sctp) /* reassign */
5023 type = CSUM_SCTP_VALID;
5026 if (!(errors & E1000_RXD_ERR_TCPE)) {
5027 mp->m_pkthdr.csum_flags |= type;
5029 mp->m_pkthdr.csum_data = htons(0xffff);
5036 * This routine is run via an vlan
5040 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5042 struct adapter *adapter = ifp->if_softc;
5045 if (ifp->if_softc != arg) /* Not our event */
5048 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5051 IGB_CORE_LOCK(adapter);
5052 index = (vtag >> 5) & 0x7F;
5054 adapter->shadow_vfta[index] |= (1 << bit);
5055 ++adapter->num_vlans;
5056 /* Change hw filter setting */
5057 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5058 igb_setup_vlan_hw_support(adapter);
5059 IGB_CORE_UNLOCK(adapter);
5063 * This routine is run via an vlan
5067 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5069 struct adapter *adapter = ifp->if_softc;
5072 if (ifp->if_softc != arg)
5075 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5078 IGB_CORE_LOCK(adapter);
5079 index = (vtag >> 5) & 0x7F;
5081 adapter->shadow_vfta[index] &= ~(1 << bit);
5082 --adapter->num_vlans;
5083 /* Change hw filter setting */
5084 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5085 igb_setup_vlan_hw_support(adapter);
5086 IGB_CORE_UNLOCK(adapter);
5090 igb_setup_vlan_hw_support(struct adapter *adapter)
5092 struct e1000_hw *hw = &adapter->hw;
5093 struct ifnet *ifp = adapter->ifp;
5096 if (adapter->vf_ifp) {
5097 e1000_rlpml_set_vf(hw,
5098 adapter->max_frame_size + VLAN_TAG_SIZE);
5102 reg = E1000_READ_REG(hw, E1000_CTRL);
5103 reg |= E1000_CTRL_VME;
5104 E1000_WRITE_REG(hw, E1000_CTRL, reg);
5106 /* Enable the Filter Table */
5107 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
5108 reg = E1000_READ_REG(hw, E1000_RCTL);
5109 reg &= ~E1000_RCTL_CFIEN;
5110 reg |= E1000_RCTL_VFE;
5111 E1000_WRITE_REG(hw, E1000_RCTL, reg);
5114 /* Update the frame size */
5115 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
5116 adapter->max_frame_size + VLAN_TAG_SIZE);
5118 /* Don't bother with table if no vlans */
5119 if ((adapter->num_vlans == 0) ||
5120 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
5123 ** A soft reset zero's out the VFTA, so
5124 ** we need to repopulate it now.
5126 for (int i = 0; i < IGB_VFTA_SIZE; i++)
5127 if (adapter->shadow_vfta[i] != 0) {
5128 if (adapter->vf_ifp)
5129 e1000_vfta_set_vf(hw,
5130 adapter->shadow_vfta[i], TRUE);
5132 e1000_write_vfta(hw,
5133 i, adapter->shadow_vfta[i]);
5138 igb_enable_intr(struct adapter *adapter)
5140 /* With RSS set up what to auto clear */
5141 if (adapter->msix_mem) {
5142 u32 mask = (adapter->que_mask | adapter->link_mask);
5143 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
5144 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
5145 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
5146 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5149 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5152 E1000_WRITE_FLUSH(&adapter->hw);
5158 igb_disable_intr(struct adapter *adapter)
5160 if (adapter->msix_mem) {
5161 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
5162 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
5164 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
5165 E1000_WRITE_FLUSH(&adapter->hw);
5170 * Bit of a misnomer, what this really means is
5171 * to enable OS management of the system... aka
5172 * to disable special hardware management features
5175 igb_init_manageability(struct adapter *adapter)
5177 if (adapter->has_manage) {
5178 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
5179 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5181 /* disable hardware interception of ARP */
5182 manc &= ~(E1000_MANC_ARP_EN);
5184 /* enable receiving management packets to the host */
5185 manc |= E1000_MANC_EN_MNG2HOST;
5186 manc2h |= 1 << 5; /* Mng Port 623 */
5187 manc2h |= 1 << 6; /* Mng Port 664 */
5188 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
5189 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5194 * Give control back to hardware management
5195 * controller if there is one.
5198 igb_release_manageability(struct adapter *adapter)
5200 if (adapter->has_manage) {
5201 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5203 /* re-enable hardware interception of ARP */
5204 manc |= E1000_MANC_ARP_EN;
5205 manc &= ~E1000_MANC_EN_MNG2HOST;
5207 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5212 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
5213 * For ASF and Pass Through versions of f/w this means that
5214 * the driver is loaded.
5218 igb_get_hw_control(struct adapter *adapter)
5222 if (adapter->vf_ifp)
5225 /* Let firmware know the driver has taken over */
5226 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5227 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5228 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
5232 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
5233 * For ASF and Pass Through versions of f/w this means that the
5234 * driver is no longer loaded.
5238 igb_release_hw_control(struct adapter *adapter)
5242 if (adapter->vf_ifp)
5245 /* Let firmware taken over control of h/w */
5246 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5247 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5248 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
5252 igb_is_valid_ether_addr(uint8_t *addr)
5254 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
5256 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
5265 * Enable PCI Wake On Lan capability
5268 igb_enable_wakeup(device_t dev)
5273 /* First find the capabilities pointer*/
5274 cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
5275 /* Read the PM Capabilities */
5276 id = pci_read_config(dev, cap, 1);
5277 if (id != PCIY_PMG) /* Something wrong */
5279 /* OK, we have the power capabilities, so
5280 now get the status register */
5281 cap += PCIR_POWER_STATUS;
5282 status = pci_read_config(dev, cap, 2);
5283 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5284 pci_write_config(dev, cap, status, 2);
5289 igb_led_func(void *arg, int onoff)
5291 struct adapter *adapter = arg;
5293 IGB_CORE_LOCK(adapter);
5295 e1000_setup_led(&adapter->hw);
5296 e1000_led_on(&adapter->hw);
5298 e1000_led_off(&adapter->hw);
5299 e1000_cleanup_led(&adapter->hw);
5301 IGB_CORE_UNLOCK(adapter);
5304 /**********************************************************************
5306 * Update the board statistics counters.
5308 **********************************************************************/
5310 igb_update_stats_counters(struct adapter *adapter)
5313 struct e1000_hw *hw = &adapter->hw;
5314 struct e1000_hw_stats *stats;
5317 ** The virtual function adapter has only a
5318 ** small controlled set of stats, do only
5319 ** those and return.
5321 if (adapter->vf_ifp) {
5322 igb_update_vf_stats_counters(adapter);
5326 stats = (struct e1000_hw_stats *)adapter->stats;
5328 if (adapter->hw.phy.media_type == e1000_media_type_copper ||
5329 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
5331 E1000_READ_REG(hw,E1000_SYMERRS);
5332 stats->sec += E1000_READ_REG(hw, E1000_SEC);
5335 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5336 stats->mpc += E1000_READ_REG(hw, E1000_MPC);
5337 stats->scc += E1000_READ_REG(hw, E1000_SCC);
5338 stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
5340 stats->mcc += E1000_READ_REG(hw, E1000_MCC);
5341 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
5342 stats->colc += E1000_READ_REG(hw, E1000_COLC);
5343 stats->dc += E1000_READ_REG(hw, E1000_DC);
5344 stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
5345 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5346 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5348 ** For watchdog management we need to know if we have been
5349 ** paused during the last interval, so capture that here.
5351 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5352 stats->xoffrxc += adapter->pause_frames;
5353 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5354 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5355 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
5356 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
5357 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
5358 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
5359 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5360 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5361 stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
5362 stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
5363 stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
5364 stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
5366 /* For the 64-bit byte counters the low dword must be read first. */
5367 /* Both registers clear on the read of the high dword */
5369 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
5370 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
5371 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
5372 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
5374 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
5375 stats->ruc += E1000_READ_REG(hw, E1000_RUC);
5376 stats->rfc += E1000_READ_REG(hw, E1000_RFC);
5377 stats->roc += E1000_READ_REG(hw, E1000_ROC);
5378 stats->rjc += E1000_READ_REG(hw, E1000_RJC);
5380 stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5381 stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC);
5382 stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5384 stats->tor += E1000_READ_REG(hw, E1000_TORL) +
5385 ((u64)E1000_READ_REG(hw, E1000_TORH) << 32);
5386 stats->tot += E1000_READ_REG(hw, E1000_TOTL) +
5387 ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32);
5389 stats->tpr += E1000_READ_REG(hw, E1000_TPR);
5390 stats->tpt += E1000_READ_REG(hw, E1000_TPT);
5391 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5392 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5393 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5394 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5395 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5396 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5397 stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
5398 stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
5400 /* Interrupt Counts */
5402 stats->iac += E1000_READ_REG(hw, E1000_IAC);
5403 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5404 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5405 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5406 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5407 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5408 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5409 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5410 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5412 /* Host to Card Statistics */
5414 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
5415 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
5416 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
5417 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
5418 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
5419 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
5420 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
5421 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
5422 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
5423 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
5424 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
5425 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
5426 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
5427 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
5429 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5430 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5431 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5432 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
5433 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5434 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5437 ifp->if_collisions = stats->colc;
5440 ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc +
5441 stats->crcerrs + stats->algnerrc +
5442 stats->ruc + stats->roc + stats->mpc + stats->cexterr;
5445 ifp->if_oerrors = stats->ecol +
5446 stats->latecol + adapter->watchdog_events;
5448 /* Driver specific counters */
5449 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
5450 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
5451 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
5452 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
5453 adapter->packet_buf_alloc_tx =
5454 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
5455 adapter->packet_buf_alloc_rx =
5456 (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
5460 /**********************************************************************
5462 * Initialize the VF board statistics counters.
5464 **********************************************************************/
5466 igb_vf_init_stats(struct adapter *adapter)
5468 struct e1000_hw *hw = &adapter->hw;
5469 struct e1000_vf_stats *stats;
5471 stats = (struct e1000_vf_stats *)adapter->stats;
5474 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
5475 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
5476 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
5477 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
5478 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
5481 /**********************************************************************
5483 * Update the VF board statistics counters.
5485 **********************************************************************/
5487 igb_update_vf_stats_counters(struct adapter *adapter)
5489 struct e1000_hw *hw = &adapter->hw;
5490 struct e1000_vf_stats *stats;
5492 if (adapter->link_speed == 0)
5495 stats = (struct e1000_vf_stats *)adapter->stats;
5497 UPDATE_VF_REG(E1000_VFGPRC,
5498 stats->last_gprc, stats->gprc);
5499 UPDATE_VF_REG(E1000_VFGORC,
5500 stats->last_gorc, stats->gorc);
5501 UPDATE_VF_REG(E1000_VFGPTC,
5502 stats->last_gptc, stats->gptc);
5503 UPDATE_VF_REG(E1000_VFGOTC,
5504 stats->last_gotc, stats->gotc);
5505 UPDATE_VF_REG(E1000_VFMPRC,
5506 stats->last_mprc, stats->mprc);
5509 /* Export a single 32-bit register via a read-only sysctl. */
5511 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5513 struct adapter *adapter;
5516 adapter = oidp->oid_arg1;
5517 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5518 return (sysctl_handle_int(oidp, &val, 0, req));
5522 ** Tuneable interrupt rate handler
5525 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5527 struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1);
5529 u32 reg, usec, rate;
5531 reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
5532 usec = ((reg & 0x7FFC) >> 2);
5534 rate = 1000000 / usec;
5537 error = sysctl_handle_int(oidp, &rate, 0, req);
5538 if (error || !req->newptr)
5544 * Add sysctl variables, one per statistic, to the system.
5547 igb_add_hw_stats(struct adapter *adapter)
5549 device_t dev = adapter->dev;
5551 struct tx_ring *txr = adapter->tx_rings;
5552 struct rx_ring *rxr = adapter->rx_rings;
5554 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5555 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5556 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5557 struct e1000_hw_stats *stats = adapter->stats;
5559 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
5560 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
5562 #define QUEUE_NAME_LEN 32
5563 char namebuf[QUEUE_NAME_LEN];
5565 /* Driver Statistics */
5566 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5567 CTLFLAG_RD, &adapter->dropped_pkts,
5568 "Driver dropped packets");
5569 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5570 CTLFLAG_RD, &adapter->link_irq,
5571 "Link MSIX IRQ Handled");
5572 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
5573 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5574 "Defragmenting mbuf chain failed");
5575 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5576 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5577 "Driver tx dma failure in xmit");
5578 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5579 CTLFLAG_RD, &adapter->rx_overruns,
5581 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5582 CTLFLAG_RD, &adapter->watchdog_events,
5583 "Watchdog timeouts");
5585 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
5586 CTLFLAG_RD, &adapter->device_control,
5587 "Device Control Register");
5588 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
5589 CTLFLAG_RD, &adapter->rx_control,
5590 "Receiver Control Register");
5591 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
5592 CTLFLAG_RD, &adapter->int_mask,
5594 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
5595 CTLFLAG_RD, &adapter->eint_mask,
5596 "Extended Interrupt Mask");
5597 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
5598 CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
5599 "Transmit Buffer Packet Allocation");
5600 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
5601 CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
5602 "Receive Buffer Packet Allocation");
5603 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5604 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5605 "Flow Control High Watermark");
5606 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5607 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5608 "Flow Control Low Watermark");
5610 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5611 struct lro_ctrl *lro = &rxr->lro;
5613 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5614 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5615 CTLFLAG_RD, NULL, "Queue Name");
5616 queue_list = SYSCTL_CHILDREN(queue_node);
5618 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5619 CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i],
5620 sizeof(&adapter->queues[i]),
5621 igb_sysctl_interrupt_rate_handler,
5622 "IU", "Interrupt Rate");
5624 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5625 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me),
5626 igb_sysctl_reg_handler, "IU",
5627 "Transmit Descriptor Head");
5628 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5629 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me),
5630 igb_sysctl_reg_handler, "IU",
5631 "Transmit Descriptor Tail");
5632 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5633 CTLFLAG_RD, &txr->no_desc_avail,
5634 "Queue No Descriptor Available");
5635 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5636 CTLFLAG_RD, &txr->total_packets,
5637 "Queue Packets Transmitted");
5639 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5640 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
5641 igb_sysctl_reg_handler, "IU",
5642 "Receive Descriptor Head");
5643 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5644 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
5645 igb_sysctl_reg_handler, "IU",
5646 "Receive Descriptor Tail");
5647 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5648 CTLFLAG_RD, &rxr->rx_packets,
5649 "Queue Packets Received");
5650 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5651 CTLFLAG_RD, &rxr->rx_bytes,
5652 "Queue Bytes Received");
5653 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
5654 CTLFLAG_RD, &lro->lro_queued, 0,
5656 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
5657 CTLFLAG_RD, &lro->lro_flushed, 0,
5661 /* MAC stats get their own sub node */
5663 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5664 CTLFLAG_RD, NULL, "MAC Statistics");
5665 stat_list = SYSCTL_CHILDREN(stat_node);
5668 ** VF adapter has a very limited set of stats
5669 ** since its not managing the metal, so to speak.
5671 if (adapter->vf_ifp) {
5672 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5673 CTLFLAG_RD, &stats->gprc,
5674 "Good Packets Received");
5675 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5676 CTLFLAG_RD, &stats->gptc,
5677 "Good Packets Transmitted");
5678 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5679 CTLFLAG_RD, &stats->gorc,
5680 "Good Octets Received");
5681 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5682 CTLFLAG_RD, &stats->gotc,
5683 "Good Octets Transmitted");
5684 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5685 CTLFLAG_RD, &stats->mprc,
5686 "Multicast Packets Received");
5690 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5691 CTLFLAG_RD, &stats->ecol,
5692 "Excessive collisions");
5693 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
5694 CTLFLAG_RD, &stats->scc,
5695 "Single collisions");
5696 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5697 CTLFLAG_RD, &stats->mcc,
5698 "Multiple collisions");
5699 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
5700 CTLFLAG_RD, &stats->latecol,
5702 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
5703 CTLFLAG_RD, &stats->colc,
5705 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5706 CTLFLAG_RD, &stats->symerrs,
5708 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5709 CTLFLAG_RD, &stats->sec,
5711 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
5712 CTLFLAG_RD, &stats->dc,
5714 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5715 CTLFLAG_RD, &stats->mpc,
5717 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors",
5718 CTLFLAG_RD, &stats->rlec,
5719 "Receive Length Errors");
5720 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5721 CTLFLAG_RD, &stats->rnbc,
5722 "Receive No Buffers");
5723 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5724 CTLFLAG_RD, &stats->ruc,
5725 "Receive Undersize");
5726 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5727 CTLFLAG_RD, &stats->rfc,
5728 "Fragmented Packets Received");
5729 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5730 CTLFLAG_RD, &stats->roc,
5731 "Oversized Packets Received");
5732 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5733 CTLFLAG_RD, &stats->rjc,
5735 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5736 CTLFLAG_RD, &stats->rxerrc,
5738 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5739 CTLFLAG_RD, &stats->crcerrs,
5741 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5742 CTLFLAG_RD, &stats->algnerrc,
5743 "Alignment Errors");
5744 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs",
5745 CTLFLAG_RD, &stats->tncrs,
5746 "Transmit with No CRS");
5747 /* On 82575 these are collision counts */
5748 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5749 CTLFLAG_RD, &stats->cexterr,
5750 "Collision/Carrier extension errors");
5751 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5752 CTLFLAG_RD, &stats->xonrxc,
5754 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5755 CTLFLAG_RD, &stats->xontxc,
5757 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5758 CTLFLAG_RD, &stats->xoffrxc,
5760 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5761 CTLFLAG_RD, &stats->xofftxc,
5762 "XOFF Transmitted");
5763 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd",
5764 CTLFLAG_RD, &stats->fcruc,
5765 "Unsupported Flow Control Received");
5766 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd",
5767 CTLFLAG_RD, &stats->mgprc,
5768 "Management Packets Received");
5769 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop",
5770 CTLFLAG_RD, &stats->mgpdc,
5771 "Management Packets Dropped");
5772 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd",
5773 CTLFLAG_RD, &stats->mgptc,
5774 "Management Packets Transmitted");
5775 /* Packet Reception Stats */
5776 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5777 CTLFLAG_RD, &stats->tpr,
5778 "Total Packets Received");
5779 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5780 CTLFLAG_RD, &stats->gprc,
5781 "Good Packets Received");
5782 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5783 CTLFLAG_RD, &stats->bprc,
5784 "Broadcast Packets Received");
5785 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5786 CTLFLAG_RD, &stats->mprc,
5787 "Multicast Packets Received");
5788 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5789 CTLFLAG_RD, &stats->prc64,
5790 "64 byte frames received");
5791 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5792 CTLFLAG_RD, &stats->prc127,
5793 "65-127 byte frames received");
5794 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5795 CTLFLAG_RD, &stats->prc255,
5796 "128-255 byte frames received");
5797 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5798 CTLFLAG_RD, &stats->prc511,
5799 "256-511 byte frames received");
5800 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5801 CTLFLAG_RD, &stats->prc1023,
5802 "512-1023 byte frames received");
5803 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5804 CTLFLAG_RD, &stats->prc1522,
5805 "1023-1522 byte frames received");
5806 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5807 CTLFLAG_RD, &stats->gorc,
5808 "Good Octets Received");
5809 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd",
5810 CTLFLAG_RD, &stats->tor,
5811 "Total Octets Received");
5813 /* Packet Transmission Stats */
5814 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5815 CTLFLAG_RD, &stats->gotc,
5816 "Good Octets Transmitted");
5817 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd",
5818 CTLFLAG_RD, &stats->tot,
5819 "Total Octets Transmitted");
5820 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5821 CTLFLAG_RD, &stats->tpt,
5822 "Total Packets Transmitted");
5823 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5824 CTLFLAG_RD, &stats->gptc,
5825 "Good Packets Transmitted");
5826 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5827 CTLFLAG_RD, &stats->bptc,
5828 "Broadcast Packets Transmitted");
5829 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5830 CTLFLAG_RD, &stats->mptc,
5831 "Multicast Packets Transmitted");
5832 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5833 CTLFLAG_RD, &stats->ptc64,
5834 "64 byte frames transmitted");
5835 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5836 CTLFLAG_RD, &stats->ptc127,
5837 "65-127 byte frames transmitted");
5838 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5839 CTLFLAG_RD, &stats->ptc255,
5840 "128-255 byte frames transmitted");
5841 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5842 CTLFLAG_RD, &stats->ptc511,
5843 "256-511 byte frames transmitted");
5844 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5845 CTLFLAG_RD, &stats->ptc1023,
5846 "512-1023 byte frames transmitted");
5847 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5848 CTLFLAG_RD, &stats->ptc1522,
5849 "1024-1522 byte frames transmitted");
5850 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5851 CTLFLAG_RD, &stats->tsctc,
5852 "TSO Contexts Transmitted");
5853 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5854 CTLFLAG_RD, &stats->tsctfc,
5855 "TSO Contexts Failed");
5858 /* Interrupt Stats */
5860 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5861 CTLFLAG_RD, NULL, "Interrupt Statistics");
5862 int_list = SYSCTL_CHILDREN(int_node);
5864 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
5865 CTLFLAG_RD, &stats->iac,
5866 "Interrupt Assertion Count");
5868 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5869 CTLFLAG_RD, &stats->icrxptc,
5870 "Interrupt Cause Rx Pkt Timer Expire Count");
5872 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5873 CTLFLAG_RD, &stats->icrxatc,
5874 "Interrupt Cause Rx Abs Timer Expire Count");
5876 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5877 CTLFLAG_RD, &stats->ictxptc,
5878 "Interrupt Cause Tx Pkt Timer Expire Count");
5880 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5881 CTLFLAG_RD, &stats->ictxatc,
5882 "Interrupt Cause Tx Abs Timer Expire Count");
5884 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5885 CTLFLAG_RD, &stats->ictxqec,
5886 "Interrupt Cause Tx Queue Empty Count");
5888 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5889 CTLFLAG_RD, &stats->ictxqmtc,
5890 "Interrupt Cause Tx Queue Min Thresh Count");
5892 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5893 CTLFLAG_RD, &stats->icrxdmtc,
5894 "Interrupt Cause Rx Desc Min Thresh Count");
5896 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5897 CTLFLAG_RD, &stats->icrxoc,
5898 "Interrupt Cause Receiver Overrun Count");
5900 /* Host to Card Stats */
5902 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
5904 "Host to Card Statistics");
5906 host_list = SYSCTL_CHILDREN(host_node);
5908 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
5909 CTLFLAG_RD, &stats->cbtmpc,
5910 "Circuit Breaker Tx Packet Count");
5912 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
5913 CTLFLAG_RD, &stats->htdpmc,
5914 "Host Transmit Discarded Packets");
5916 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
5917 CTLFLAG_RD, &stats->rpthc,
5918 "Rx Packets To Host");
5920 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
5921 CTLFLAG_RD, &stats->cbrmpc,
5922 "Circuit Breaker Rx Packet Count");
5924 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
5925 CTLFLAG_RD, &stats->cbrdpc,
5926 "Circuit Breaker Rx Dropped Count");
5928 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
5929 CTLFLAG_RD, &stats->hgptc,
5930 "Host Good Packets Tx Count");
5932 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
5933 CTLFLAG_RD, &stats->htcbdpc,
5934 "Host Tx Circuit Breaker Dropped Count");
5936 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
5937 CTLFLAG_RD, &stats->hgorc,
5938 "Host Good Octets Received Count");
5940 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
5941 CTLFLAG_RD, &stats->hgotc,
5942 "Host Good Octets Transmit Count");
5944 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
5945 CTLFLAG_RD, &stats->lenerrs,
5948 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
5949 CTLFLAG_RD, &stats->scvpc,
5950 "SerDes/SGMII Code Violation Pkt Count");
5952 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
5953 CTLFLAG_RD, &stats->hrmpc,
5954 "Header Redirection Missed Packet Count");
5958 /**********************************************************************
5960 * This routine provides a way to dump out the adapter eeprom,
5961 * often a useful debug/service tool. This only dumps the first
5962 * 32 words, stuff that matters is in that extent.
5964 **********************************************************************/
5966 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5968 struct adapter *adapter;
5973 error = sysctl_handle_int(oidp, &result, 0, req);
5975 if (error || !req->newptr)
5979 * This value will cause a hex dump of the
5980 * first 32 16-bit words of the EEPROM to
5984 adapter = (struct adapter *)arg1;
5985 igb_print_nvm_info(adapter);
5992 igb_print_nvm_info(struct adapter *adapter)
5997 /* Its a bit crude, but it gets the job done */
5998 printf("\nInterface EEPROM Dump:\n");
5999 printf("Offset\n0x0000 ");
6000 for (i = 0, j = 0; i < 32; i++, j++) {
6001 if (j == 8) { /* Make the offset block */
6003 printf("\n0x00%x0 ",row);
6005 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
6006 printf("%04x ", eeprom_data);
6012 igb_set_sysctl_value(struct adapter *adapter, const char *name,
6013 const char *description, int *limit, int value)
6016 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
6017 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
6018 OID_AUTO, name, CTLFLAG_RW, limit, value, description);
6022 ** Set flow control using sysctl:
6023 ** Flow control values:
6030 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
6033 static int input = 3; /* default is full */
6034 struct adapter *adapter = (struct adapter *) arg1;
6036 error = sysctl_handle_int(oidp, &input, 0, req);
6038 if ((error) || (req->newptr == NULL))
6042 case e1000_fc_rx_pause:
6043 case e1000_fc_tx_pause:
6046 adapter->hw.fc.requested_mode = input;
6047 adapter->fc = input;
6054 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
6055 e1000_force_mac_fc(&adapter->hw);
6056 /* XXX TODO: update DROP_EN on each RX queue if appropriate */
6061 ** Manage DMA Coalesce:
6064 ** Legal timer values are:
6065 ** 250,500,1000-10000 in thousands
6068 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
6070 struct adapter *adapter = (struct adapter *) arg1;
6073 error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
6075 if ((error) || (req->newptr == NULL))
6078 switch (adapter->dmac) {
6082 case 1: /* Just enable and use default */
6083 adapter->dmac = 1000;
6097 /* Legal values - allow */
6100 /* Do nothing, illegal value */
6104 /* Reinit the interface */
6110 ** Manage Energy Efficient Ethernet:
6112 ** 0/1 - enabled/disabled
6115 igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
6117 struct adapter *adapter = (struct adapter *) arg1;
6120 value = adapter->hw.dev_spec._82575.eee_disable;
6121 error = sysctl_handle_int(oidp, &value, 0, req);
6122 if (error || req->newptr == NULL)
6124 IGB_CORE_LOCK(adapter);
6125 adapter->hw.dev_spec._82575.eee_disable = (value != 0);
6126 igb_init_locked(adapter);
6127 IGB_CORE_UNLOCK(adapter);