1 /******************************************************************************
3 Copyright (c) 2001-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
40 #ifdef HAVE_KERNEL_OPTION_HEADERS
41 #include "opt_device_polling.h"
47 /*********************************************************************
49 *********************************************************************/
50 char igb_driver_version[] = "2.5.3-k";
53 /*********************************************************************
56 * Used by probe to select devices to load on
57 * Last field stores an index into e1000_strings
58 * Last entry must be all 0s
60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61 *********************************************************************/
63 static igb_vendor_info_t igb_vendor_info_array[] =
65 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0},
66 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0},
67 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0},
68 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0},
69 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0},
70 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0},
71 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0},
72 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0},
73 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0},
74 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0},
75 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0},
76 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0},
77 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0},
78 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0},
79 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0},
80 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0},
81 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0},
82 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0},
83 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0},
84 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0},
85 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0},
86 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0},
87 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0},
88 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0},
89 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0},
90 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0},
91 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0},
92 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0},
93 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0},
94 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0},
95 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0},
96 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0},
97 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0},
98 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0},
99 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0},
100 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0},
101 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0},
102 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0},
103 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0},
104 /* required last entry */
108 /*********************************************************************
109 * Table of branding strings for all supported NICs.
110 *********************************************************************/
112 static char *igb_strings[] = {
113 "Intel(R) PRO/1000 Network Connection"
116 /*********************************************************************
117 * Function prototypes
118 *********************************************************************/
119 static int igb_probe(device_t);
120 static int igb_attach(device_t);
121 static int igb_detach(device_t);
122 static int igb_shutdown(device_t);
123 static int igb_suspend(device_t);
124 static int igb_resume(device_t);
125 #ifndef IGB_LEGACY_TX
126 static int igb_mq_start(struct ifnet *, struct mbuf *);
127 static int igb_mq_start_locked(struct ifnet *, struct tx_ring *);
128 static void igb_qflush(struct ifnet *);
129 static void igb_deferred_mq_start(void *, int);
131 static void igb_start(struct ifnet *);
132 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
134 static int igb_ioctl(struct ifnet *, u_long, caddr_t);
135 static uint64_t igb_get_counter(if_t, ift_counter);
136 static void igb_init(void *);
137 static void igb_init_locked(struct adapter *);
138 static void igb_stop(void *);
139 static void igb_media_status(struct ifnet *, struct ifmediareq *);
140 static int igb_media_change(struct ifnet *);
141 static void igb_identify_hardware(struct adapter *);
142 static int igb_allocate_pci_resources(struct adapter *);
143 static int igb_allocate_msix(struct adapter *);
144 static int igb_allocate_legacy(struct adapter *);
145 static int igb_setup_msix(struct adapter *);
146 static void igb_free_pci_resources(struct adapter *);
147 static void igb_local_timer(void *);
148 static void igb_reset(struct adapter *);
149 static int igb_setup_interface(device_t, struct adapter *);
150 static int igb_allocate_queues(struct adapter *);
151 static void igb_configure_queues(struct adapter *);
153 static int igb_allocate_transmit_buffers(struct tx_ring *);
154 static void igb_setup_transmit_structures(struct adapter *);
155 static void igb_setup_transmit_ring(struct tx_ring *);
156 static void igb_initialize_transmit_units(struct adapter *);
157 static void igb_free_transmit_structures(struct adapter *);
158 static void igb_free_transmit_buffers(struct tx_ring *);
160 static int igb_allocate_receive_buffers(struct rx_ring *);
161 static int igb_setup_receive_structures(struct adapter *);
162 static int igb_setup_receive_ring(struct rx_ring *);
163 static void igb_initialize_receive_units(struct adapter *);
164 static void igb_free_receive_structures(struct adapter *);
165 static void igb_free_receive_buffers(struct rx_ring *);
166 static void igb_free_receive_ring(struct rx_ring *);
168 static void igb_enable_intr(struct adapter *);
169 static void igb_disable_intr(struct adapter *);
170 static void igb_update_stats_counters(struct adapter *);
171 static bool igb_txeof(struct tx_ring *);
173 static __inline void igb_rx_discard(struct rx_ring *, int);
174 static __inline void igb_rx_input(struct rx_ring *,
175 struct ifnet *, struct mbuf *, u32);
177 static bool igb_rxeof(struct igb_queue *, int, int *);
178 static void igb_rx_checksum(u32, struct mbuf *, u32);
179 static int igb_tx_ctx_setup(struct tx_ring *,
180 struct mbuf *, u32 *, u32 *);
181 static int igb_tso_setup(struct tx_ring *,
182 struct mbuf *, u32 *, u32 *);
183 static void igb_set_promisc(struct adapter *);
184 static void igb_disable_promisc(struct adapter *);
185 static void igb_set_multi(struct adapter *);
186 static void igb_update_link_status(struct adapter *);
187 static void igb_refresh_mbufs(struct rx_ring *, int);
189 static void igb_register_vlan(void *, struct ifnet *, u16);
190 static void igb_unregister_vlan(void *, struct ifnet *, u16);
191 static void igb_setup_vlan_hw_support(struct adapter *);
193 static int igb_xmit(struct tx_ring *, struct mbuf **);
194 static int igb_dma_malloc(struct adapter *, bus_size_t,
195 struct igb_dma_alloc *, int);
196 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
197 static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
198 static void igb_print_nvm_info(struct adapter *);
199 static int igb_is_valid_ether_addr(u8 *);
200 static void igb_add_hw_stats(struct adapter *);
202 static void igb_vf_init_stats(struct adapter *);
203 static void igb_update_vf_stats_counters(struct adapter *);
205 /* Management and WOL Support */
206 static void igb_init_manageability(struct adapter *);
207 static void igb_release_manageability(struct adapter *);
208 static void igb_get_hw_control(struct adapter *);
209 static void igb_release_hw_control(struct adapter *);
210 static void igb_enable_wakeup(device_t);
211 static void igb_led_func(void *, int);
213 static int igb_irq_fast(void *);
214 static void igb_msix_que(void *);
215 static void igb_msix_link(void *);
216 static void igb_handle_que(void *context, int pending);
217 static void igb_handle_link(void *context, int pending);
218 static void igb_handle_link_locked(struct adapter *);
220 static void igb_set_sysctl_value(struct adapter *, const char *,
221 const char *, int *, int);
222 static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
223 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
224 static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
226 #ifdef DEVICE_POLLING
227 static poll_handler_t igb_poll;
230 /*********************************************************************
231 * FreeBSD Device Interface Entry Points
232 *********************************************************************/
234 static device_method_t igb_methods[] = {
235 /* Device interface */
236 DEVMETHOD(device_probe, igb_probe),
237 DEVMETHOD(device_attach, igb_attach),
238 DEVMETHOD(device_detach, igb_detach),
239 DEVMETHOD(device_shutdown, igb_shutdown),
240 DEVMETHOD(device_suspend, igb_suspend),
241 DEVMETHOD(device_resume, igb_resume),
245 static driver_t igb_driver = {
246 "igb", igb_methods, sizeof(struct adapter),
249 static devclass_t igb_devclass;
250 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
251 MODULE_DEPEND(igb, pci, 1, 1, 1);
252 MODULE_DEPEND(igb, ether, 1, 1, 1);
254 MODULE_DEPEND(igb, netmap, 1, 1, 1);
255 #endif /* DEV_NETMAP */
257 /*********************************************************************
258 * Tunable default values.
259 *********************************************************************/
261 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
263 /* Descriptor defaults */
264 static int igb_rxd = IGB_DEFAULT_RXD;
265 static int igb_txd = IGB_DEFAULT_TXD;
266 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
267 "Number of receive descriptors per queue");
268 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
269 "Number of transmit descriptors per queue");
272 ** AIM: Adaptive Interrupt Moderation
273 ** which means that the interrupt rate
274 ** is varied over time based on the
275 ** traffic for that interrupt vector
277 static int igb_enable_aim = TRUE;
278 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0,
279 "Enable adaptive interrupt moderation");
282 * MSIX should be the default for best performance,
283 * but this allows it to be forced off for testing.
285 static int igb_enable_msix = 1;
286 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
287 "Enable MSI-X interrupts");
290 ** Tuneable Interrupt rate
292 static int igb_max_interrupt_rate = 8000;
293 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
294 &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
296 #ifndef IGB_LEGACY_TX
298 ** Tuneable number of buffers in the buf-ring (drbr_xxx)
300 static int igb_buf_ring_size = IGB_BR_SIZE;
301 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
302 &igb_buf_ring_size, 0, "Size of the bufring");
306 ** Header split causes the packet header to
307 ** be dma'd to a separate mbuf from the payload.
308 ** this can have memory alignment benefits. But
309 ** another plus is that small packets often fit
310 ** into the header and thus use no cluster. Its
311 ** a very workload dependent type feature.
313 static int igb_header_split = FALSE;
314 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
315 "Enable receive mbuf header split");
318 ** This will autoconfigure based on the
319 ** number of CPUs and max supported
320 ** MSIX messages if left at 0.
322 static int igb_num_queues = 0;
323 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
324 "Number of queues to configure, 0 indicates autoconfigure");
327 ** Global variable to store last used CPU when binding queues
328 ** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
329 ** queue is bound to a cpu.
331 static int igb_last_bind_cpu = -1;
333 /* How many packets rxeof tries to clean at a time */
334 static int igb_rx_process_limit = 100;
335 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
336 &igb_rx_process_limit, 0,
337 "Maximum number of received packets to process at a time, -1 means unlimited");
339 /* How many packets txeof tries to clean at a time */
340 static int igb_tx_process_limit = -1;
341 SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
342 &igb_tx_process_limit, 0,
343 "Maximum number of sent packets to process at a time, -1 means unlimited");
345 #ifdef DEV_NETMAP /* see ixgbe.c for details */
346 #include <dev/netmap/if_igb_netmap.h>
347 #endif /* DEV_NETMAP */
348 /*********************************************************************
349 * Device identification routine
351 * igb_probe determines if the driver should be loaded on
352 * adapter based on PCI vendor/device id of the adapter.
354 * return BUS_PROBE_DEFAULT on success, positive on failure
355 *********************************************************************/
358 igb_probe(device_t dev)
360 char adapter_name[256];
361 uint16_t pci_vendor_id = 0;
362 uint16_t pci_device_id = 0;
363 uint16_t pci_subvendor_id = 0;
364 uint16_t pci_subdevice_id = 0;
365 igb_vendor_info_t *ent;
367 INIT_DEBUGOUT("igb_probe: begin");
369 pci_vendor_id = pci_get_vendor(dev);
370 if (pci_vendor_id != IGB_INTEL_VENDOR_ID)
373 pci_device_id = pci_get_device(dev);
374 pci_subvendor_id = pci_get_subvendor(dev);
375 pci_subdevice_id = pci_get_subdevice(dev);
377 ent = igb_vendor_info_array;
378 while (ent->vendor_id != 0) {
379 if ((pci_vendor_id == ent->vendor_id) &&
380 (pci_device_id == ent->device_id) &&
382 ((pci_subvendor_id == ent->subvendor_id) ||
383 (ent->subvendor_id == 0)) &&
385 ((pci_subdevice_id == ent->subdevice_id) ||
386 (ent->subdevice_id == 0))) {
387 sprintf(adapter_name, "%s, Version - %s",
388 igb_strings[ent->index],
390 device_set_desc_copy(dev, adapter_name);
391 return (BUS_PROBE_DEFAULT);
398 /*********************************************************************
399 * Device initialization routine
401 * The attach entry point is called when the driver is being loaded.
402 * This routine identifies the type of hardware, allocates all resources
403 * and initializes the hardware.
405 * return 0 on success, positive on failure
406 *********************************************************************/
409 igb_attach(device_t dev)
411 struct adapter *adapter;
415 INIT_DEBUGOUT("igb_attach: begin");
417 if (resource_disabled("igb", device_get_unit(dev))) {
418 device_printf(dev, "Disabled by device hint\n");
422 adapter = device_get_softc(dev);
423 adapter->dev = adapter->osdep.dev = dev;
424 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
427 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
428 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
429 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
430 igb_sysctl_nvm_info, "I", "NVM Information");
432 igb_set_sysctl_value(adapter, "enable_aim",
433 "Interrupt Moderation", &adapter->enable_aim,
436 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
437 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
438 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
439 adapter, 0, igb_set_flowcntl, "I", "Flow Control");
441 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
443 /* Determine hardware and mac info */
444 igb_identify_hardware(adapter);
446 /* Setup PCI resources */
447 if (igb_allocate_pci_resources(adapter)) {
448 device_printf(dev, "Allocation of PCI resources failed\n");
453 /* Do Shared Code initialization */
454 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
455 device_printf(dev, "Setup of Shared code failed\n");
460 e1000_get_bus_info(&adapter->hw);
462 /* Sysctls for limiting the amount of work done in the taskqueues */
463 igb_set_sysctl_value(adapter, "rx_processing_limit",
464 "max number of rx packets to process",
465 &adapter->rx_process_limit, igb_rx_process_limit);
467 igb_set_sysctl_value(adapter, "tx_processing_limit",
468 "max number of tx packets to process",
469 &adapter->tx_process_limit, igb_tx_process_limit);
472 * Validate number of transmit and receive descriptors. It
473 * must not exceed hardware maximum, and must be multiple
474 * of E1000_DBA_ALIGN.
476 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
477 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
478 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
479 IGB_DEFAULT_TXD, igb_txd);
480 adapter->num_tx_desc = IGB_DEFAULT_TXD;
482 adapter->num_tx_desc = igb_txd;
483 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
484 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
485 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
486 IGB_DEFAULT_RXD, igb_rxd);
487 adapter->num_rx_desc = IGB_DEFAULT_RXD;
489 adapter->num_rx_desc = igb_rxd;
491 adapter->hw.mac.autoneg = DO_AUTO_NEG;
492 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
493 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
496 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
497 adapter->hw.phy.mdix = AUTO_ALL_MODES;
498 adapter->hw.phy.disable_polarity_correction = FALSE;
499 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
503 * Set the frame limits assuming
504 * standard ethernet sized frames.
506 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
509 ** Allocate and Setup Queues
511 if (igb_allocate_queues(adapter)) {
516 /* Allocate the appropriate stats memory */
517 if (adapter->vf_ifp) {
519 (struct e1000_vf_stats *)malloc(sizeof \
520 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
521 igb_vf_init_stats(adapter);
524 (struct e1000_hw_stats *)malloc(sizeof \
525 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
526 if (adapter->stats == NULL) {
527 device_printf(dev, "Can not allocate stats memory\n");
532 /* Allocate multicast array memory. */
533 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
534 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
535 if (adapter->mta == NULL) {
536 device_printf(dev, "Can not allocate multicast setup array\n");
541 /* Some adapter-specific advanced features */
542 if (adapter->hw.mac.type >= e1000_i350) {
543 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
544 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
545 OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
546 adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
547 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
548 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
549 OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
550 adapter, 0, igb_sysctl_eee, "I",
551 "Disable Energy Efficient Ethernet");
552 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
553 if (adapter->hw.mac.type == e1000_i354)
554 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
556 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
561 ** Start from a known state, this is
562 ** important in reading the nvm and
565 e1000_reset_hw(&adapter->hw);
567 /* Make sure we have a good EEPROM before we read from it */
568 if (((adapter->hw.mac.type != e1000_i210) &&
569 (adapter->hw.mac.type != e1000_i211)) &&
570 (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
572 ** Some PCI-E parts fail the first check due to
573 ** the link being in sleep state, call it again,
574 ** if it fails a second time its a real issue.
576 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
578 "The EEPROM Checksum Is Not Valid\n");
585 ** Copy the permanent MAC address out of the EEPROM
587 if (e1000_read_mac_addr(&adapter->hw) < 0) {
588 device_printf(dev, "EEPROM read error while reading MAC"
594 /* Check its sanity */
595 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
596 if (adapter->vf_ifp) {
597 u8 addr[ETHER_ADDR_LEN];
598 arc4rand(&addr, sizeof(addr), 0);
601 bcopy(addr, adapter->hw.mac.addr, sizeof(addr));
603 device_printf(dev, "Invalid MAC address\n");
609 /* Setup OS specific network interface */
610 if (igb_setup_interface(dev, adapter) != 0)
613 /* Now get a good starting state */
616 /* Initialize statistics */
617 igb_update_stats_counters(adapter);
619 adapter->hw.mac.get_link_status = 1;
620 igb_update_link_status(adapter);
622 /* Indicate SOL/IDER usage */
623 if (e1000_check_reset_block(&adapter->hw))
625 "PHY reset is blocked due to SOL/IDER session.\n");
627 /* Determine if we have to control management hardware */
628 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
633 /* APME bit in EEPROM is mapped to WUC.APME */
634 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
636 adapter->wol = E1000_WUFC_MAG;
638 /* Register for VLAN events */
639 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
640 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
641 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
642 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
644 igb_add_hw_stats(adapter);
646 /* Tell the stack that the interface is not active */
647 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
648 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
650 adapter->led_dev = led_create(igb_led_func, adapter,
651 device_get_nameunit(dev));
654 ** Configure Interrupts
656 if ((adapter->msix > 1) && (igb_enable_msix))
657 error = igb_allocate_msix(adapter);
658 else /* MSI or Legacy */
659 error = igb_allocate_legacy(adapter);
664 igb_netmap_attach(adapter);
665 #endif /* DEV_NETMAP */
666 INIT_DEBUGOUT("igb_attach: end");
671 if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */
673 igb_free_transmit_structures(adapter);
674 igb_free_receive_structures(adapter);
675 igb_release_hw_control(adapter);
677 igb_free_pci_resources(adapter);
678 if (adapter->ifp != NULL)
679 if_free(adapter->ifp);
680 free(adapter->mta, M_DEVBUF);
681 IGB_CORE_LOCK_DESTROY(adapter);
686 /*********************************************************************
687 * Device removal routine
689 * The detach entry point is called when the driver is being removed.
690 * This routine stops the adapter and deallocates all the resources
691 * that were allocated for driver operation.
693 * return 0 on success, positive on failure
694 *********************************************************************/
697 igb_detach(device_t dev)
699 struct adapter *adapter = device_get_softc(dev);
700 struct ifnet *ifp = adapter->ifp;
702 INIT_DEBUGOUT("igb_detach: begin");
704 /* Make sure VLANS are not using driver */
705 if (adapter->ifp->if_vlantrunk != NULL) {
706 device_printf(dev,"Vlan in use, detach first\n");
710 ether_ifdetach(adapter->ifp);
712 if (adapter->led_dev != NULL)
713 led_destroy(adapter->led_dev);
715 #ifdef DEVICE_POLLING
716 if (ifp->if_capenable & IFCAP_POLLING)
717 ether_poll_deregister(ifp);
720 IGB_CORE_LOCK(adapter);
721 adapter->in_detach = 1;
723 IGB_CORE_UNLOCK(adapter);
725 e1000_phy_hw_reset(&adapter->hw);
727 /* Give control back to firmware */
728 igb_release_manageability(adapter);
729 igb_release_hw_control(adapter);
732 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
733 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
734 igb_enable_wakeup(dev);
737 /* Unregister VLAN events */
738 if (adapter->vlan_attach != NULL)
739 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
740 if (adapter->vlan_detach != NULL)
741 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
743 callout_drain(&adapter->timer);
746 netmap_detach(adapter->ifp);
747 #endif /* DEV_NETMAP */
748 igb_free_pci_resources(adapter);
749 bus_generic_detach(dev);
752 igb_free_transmit_structures(adapter);
753 igb_free_receive_structures(adapter);
754 if (adapter->mta != NULL)
755 free(adapter->mta, M_DEVBUF);
757 IGB_CORE_LOCK_DESTROY(adapter);
762 /*********************************************************************
764 * Shutdown entry point
766 **********************************************************************/
769 igb_shutdown(device_t dev)
771 return igb_suspend(dev);
775 * Suspend/resume device methods.
778 igb_suspend(device_t dev)
780 struct adapter *adapter = device_get_softc(dev);
782 IGB_CORE_LOCK(adapter);
786 igb_release_manageability(adapter);
787 igb_release_hw_control(adapter);
790 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
791 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
792 igb_enable_wakeup(dev);
795 IGB_CORE_UNLOCK(adapter);
797 return bus_generic_suspend(dev);
801 igb_resume(device_t dev)
803 struct adapter *adapter = device_get_softc(dev);
804 struct tx_ring *txr = adapter->tx_rings;
805 struct ifnet *ifp = adapter->ifp;
807 IGB_CORE_LOCK(adapter);
808 igb_init_locked(adapter);
809 igb_init_manageability(adapter);
811 if ((ifp->if_flags & IFF_UP) &&
812 (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
813 for (int i = 0; i < adapter->num_queues; i++, txr++) {
815 #ifndef IGB_LEGACY_TX
816 /* Process the stack queue only if not depleted */
817 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
818 !drbr_empty(ifp, txr->br))
819 igb_mq_start_locked(ifp, txr);
821 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
822 igb_start_locked(txr, ifp);
827 IGB_CORE_UNLOCK(adapter);
829 return bus_generic_resume(dev);
835 /*********************************************************************
836 * Transmit entry point
838 * igb_start is called by the stack to initiate a transmit.
839 * The driver will remain in this routine as long as there are
840 * packets to transmit and transmit resources are available.
841 * In case resources are not available stack is notified and
842 * the packet is requeued.
843 **********************************************************************/
846 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
848 struct adapter *adapter = ifp->if_softc;
851 IGB_TX_LOCK_ASSERT(txr);
853 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
856 if (!adapter->link_active)
859 /* Call cleanup if number of TX descriptors low */
860 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
863 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
864 if (txr->tx_avail <= IGB_MAX_SCATTER) {
865 txr->queue_status |= IGB_QUEUE_DEPLETED;
868 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
872 * Encapsulation can modify our pointer, and or make it
873 * NULL on failure. In that event, we can't requeue.
875 if (igb_xmit(txr, &m_head)) {
877 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
878 if (txr->tx_avail <= IGB_MAX_SCATTER)
879 txr->queue_status |= IGB_QUEUE_DEPLETED;
883 /* Send a copy of the frame to the BPF listener */
884 ETHER_BPF_MTAP(ifp, m_head);
886 /* Set watchdog on */
887 txr->watchdog_time = ticks;
888 txr->queue_status |= IGB_QUEUE_WORKING;
893 * Legacy TX driver routine, called from the
894 * stack, always uses tx[0], and spins for it.
895 * Should not be used with multiqueue tx
898 igb_start(struct ifnet *ifp)
900 struct adapter *adapter = ifp->if_softc;
901 struct tx_ring *txr = adapter->tx_rings;
903 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
905 igb_start_locked(txr, ifp);
911 #else /* ~IGB_LEGACY_TX */
914 ** Multiqueue Transmit Entry:
915 ** quick turnaround to the stack
919 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
921 struct adapter *adapter = ifp->if_softc;
922 struct igb_queue *que;
929 /* Which queue to use */
931 * When doing RSS, map it to the same outbound queue
932 * as the incoming flow would be mapped to.
934 * If everything is setup correctly, it should be the
935 * same bucket that the current CPU we're on is.
937 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
939 if (rss_hash2bucket(m->m_pkthdr.flowid,
940 M_HASHTYPE_GET(m), &bucket_id) == 0) {
941 /* XXX TODO: spit out something if bucket_id > num_queues? */
942 i = bucket_id % adapter->num_queues;
945 i = m->m_pkthdr.flowid % adapter->num_queues;
950 i = curcpu % adapter->num_queues;
952 txr = &adapter->tx_rings[i];
953 que = &adapter->queues[i];
955 err = drbr_enqueue(ifp, txr->br, m);
958 if (IGB_TX_TRYLOCK(txr)) {
959 igb_mq_start_locked(ifp, txr);
962 taskqueue_enqueue(que->tq, &txr->txq_task);
968 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
970 struct adapter *adapter = txr->adapter;
972 int err = 0, enq = 0;
974 IGB_TX_LOCK_ASSERT(txr);
976 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
977 adapter->link_active == 0)
980 /* Process the queue */
981 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
982 if ((err = igb_xmit(txr, &next)) != 0) {
984 /* It was freed, move forward */
985 drbr_advance(ifp, txr->br);
988 * Still have one left, it may not be
989 * the same since the transmit function
990 * may have changed it.
992 drbr_putback(ifp, txr->br, next);
996 drbr_advance(ifp, txr->br);
998 if (next->m_flags & M_MCAST && adapter->vf_ifp)
999 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
1000 ETHER_BPF_MTAP(ifp, next);
1001 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1005 /* Set the watchdog */
1006 txr->queue_status |= IGB_QUEUE_WORKING;
1007 txr->watchdog_time = ticks;
1009 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
1011 if (txr->tx_avail <= IGB_MAX_SCATTER)
1012 txr->queue_status |= IGB_QUEUE_DEPLETED;
1017 * Called from a taskqueue to drain queued transmit packets.
1020 igb_deferred_mq_start(void *arg, int pending)
1022 struct tx_ring *txr = arg;
1023 struct adapter *adapter = txr->adapter;
1024 struct ifnet *ifp = adapter->ifp;
1027 if (!drbr_empty(ifp, txr->br))
1028 igb_mq_start_locked(ifp, txr);
1033 ** Flush all ring buffers
1036 igb_qflush(struct ifnet *ifp)
1038 struct adapter *adapter = ifp->if_softc;
1039 struct tx_ring *txr = adapter->tx_rings;
1042 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1044 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1050 #endif /* ~IGB_LEGACY_TX */
1052 /*********************************************************************
1055 * igb_ioctl is called when the user wants to configure the
1058 * return 0 on success, positive on failure
1059 **********************************************************************/
1062 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1064 struct adapter *adapter = ifp->if_softc;
1065 struct ifreq *ifr = (struct ifreq *)data;
1066 #if defined(INET) || defined(INET6)
1067 struct ifaddr *ifa = (struct ifaddr *)data;
1069 bool avoid_reset = FALSE;
1072 if (adapter->in_detach)
1078 if (ifa->ifa_addr->sa_family == AF_INET)
1082 if (ifa->ifa_addr->sa_family == AF_INET6)
1086 ** Calling init results in link renegotiation,
1087 ** so we avoid doing it when possible.
1090 ifp->if_flags |= IFF_UP;
1091 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1094 if (!(ifp->if_flags & IFF_NOARP))
1095 arp_ifinit(ifp, ifa);
1098 error = ether_ioctl(ifp, command, data);
1104 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1106 IGB_CORE_LOCK(adapter);
1107 max_frame_size = 9234;
1108 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1110 IGB_CORE_UNLOCK(adapter);
1115 ifp->if_mtu = ifr->ifr_mtu;
1116 adapter->max_frame_size =
1117 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1118 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1119 igb_init_locked(adapter);
1120 IGB_CORE_UNLOCK(adapter);
1124 IOCTL_DEBUGOUT("ioctl rcv'd:\
1125 SIOCSIFFLAGS (Set Interface Flags)");
1126 IGB_CORE_LOCK(adapter);
1127 if (ifp->if_flags & IFF_UP) {
1128 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1129 if ((ifp->if_flags ^ adapter->if_flags) &
1130 (IFF_PROMISC | IFF_ALLMULTI)) {
1131 igb_disable_promisc(adapter);
1132 igb_set_promisc(adapter);
1135 igb_init_locked(adapter);
1137 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1139 adapter->if_flags = ifp->if_flags;
1140 IGB_CORE_UNLOCK(adapter);
1144 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1145 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1146 IGB_CORE_LOCK(adapter);
1147 igb_disable_intr(adapter);
1148 igb_set_multi(adapter);
1149 #ifdef DEVICE_POLLING
1150 if (!(ifp->if_capenable & IFCAP_POLLING))
1152 igb_enable_intr(adapter);
1153 IGB_CORE_UNLOCK(adapter);
1157 /* Check SOL/IDER usage */
1158 IGB_CORE_LOCK(adapter);
1159 if (e1000_check_reset_block(&adapter->hw)) {
1160 IGB_CORE_UNLOCK(adapter);
1161 device_printf(adapter->dev, "Media change is"
1162 " blocked due to SOL/IDER session.\n");
1165 IGB_CORE_UNLOCK(adapter);
1167 IOCTL_DEBUGOUT("ioctl rcv'd: \
1168 SIOCxIFMEDIA (Get/Set Interface Media)");
1169 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1175 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1177 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1178 #ifdef DEVICE_POLLING
1179 if (mask & IFCAP_POLLING) {
1180 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1181 error = ether_poll_register(igb_poll, ifp);
1184 IGB_CORE_LOCK(adapter);
1185 igb_disable_intr(adapter);
1186 ifp->if_capenable |= IFCAP_POLLING;
1187 IGB_CORE_UNLOCK(adapter);
1189 error = ether_poll_deregister(ifp);
1190 /* Enable interrupt even in error case */
1191 IGB_CORE_LOCK(adapter);
1192 igb_enable_intr(adapter);
1193 ifp->if_capenable &= ~IFCAP_POLLING;
1194 IGB_CORE_UNLOCK(adapter);
1198 #if __FreeBSD_version >= 1000000
1199 /* HW cannot turn these on/off separately */
1200 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
1201 ifp->if_capenable ^= IFCAP_RXCSUM;
1202 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1205 if (mask & IFCAP_TXCSUM) {
1206 ifp->if_capenable ^= IFCAP_TXCSUM;
1209 if (mask & IFCAP_TXCSUM_IPV6) {
1210 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1214 if (mask & IFCAP_HWCSUM) {
1215 ifp->if_capenable ^= IFCAP_HWCSUM;
1219 if (mask & IFCAP_TSO4) {
1220 ifp->if_capenable ^= IFCAP_TSO4;
1223 if (mask & IFCAP_TSO6) {
1224 ifp->if_capenable ^= IFCAP_TSO6;
1227 if (mask & IFCAP_VLAN_HWTAGGING) {
1228 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1231 if (mask & IFCAP_VLAN_HWFILTER) {
1232 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1235 if (mask & IFCAP_VLAN_HWTSO) {
1236 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1239 if (mask & IFCAP_LRO) {
1240 ifp->if_capenable ^= IFCAP_LRO;
1243 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1245 VLAN_CAPABILITIES(ifp);
1250 error = ether_ioctl(ifp, command, data);
1258 /*********************************************************************
1261 * This routine is used in two ways. It is used by the stack as
1262 * init entry point in network interface structure. It is also used
1263 * by the driver as a hw/sw initialization routine to get to a
1266 * return 0 on success, positive on failure
1267 **********************************************************************/
1270 igb_init_locked(struct adapter *adapter)
1272 struct ifnet *ifp = adapter->ifp;
1273 device_t dev = adapter->dev;
1275 INIT_DEBUGOUT("igb_init: begin");
1277 IGB_CORE_LOCK_ASSERT(adapter);
1279 igb_disable_intr(adapter);
1280 callout_stop(&adapter->timer);
1282 /* Get the latest mac address, User can use a LAA */
1283 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1286 /* Put the address into the Receive Address Array */
1287 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1290 igb_update_link_status(adapter);
1292 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1294 /* Set hardware offload abilities */
1295 ifp->if_hwassist = 0;
1296 if (ifp->if_capenable & IFCAP_TXCSUM) {
1297 #if __FreeBSD_version >= 1000000
1298 ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP);
1299 if (adapter->hw.mac.type != e1000_82575)
1300 ifp->if_hwassist |= CSUM_IP_SCTP;
1302 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1303 #if __FreeBSD_version >= 800000
1304 if (adapter->hw.mac.type != e1000_82575)
1305 ifp->if_hwassist |= CSUM_SCTP;
1310 #if __FreeBSD_version >= 1000000
1311 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) {
1312 ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP);
1313 if (adapter->hw.mac.type != e1000_82575)
1314 ifp->if_hwassist |= CSUM_IP6_SCTP;
1317 if (ifp->if_capenable & IFCAP_TSO)
1318 ifp->if_hwassist |= CSUM_TSO;
1320 /* Clear bad data from Rx FIFOs */
1321 e1000_rx_fifo_flush_82575(&adapter->hw);
1323 /* Configure for OS presence */
1324 igb_init_manageability(adapter);
1326 /* Prepare transmit descriptors and buffers */
1327 igb_setup_transmit_structures(adapter);
1328 igb_initialize_transmit_units(adapter);
1330 /* Setup Multicast table */
1331 igb_set_multi(adapter);
1334 ** Figure out the desired mbuf pool
1335 ** for doing jumbo/packetsplit
1337 if (adapter->max_frame_size <= 2048)
1338 adapter->rx_mbuf_sz = MCLBYTES;
1339 else if (adapter->max_frame_size <= 4096)
1340 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1342 adapter->rx_mbuf_sz = MJUM9BYTES;
1344 /* Prepare receive descriptors and buffers */
1345 if (igb_setup_receive_structures(adapter)) {
1346 device_printf(dev, "Could not setup receive structures\n");
1349 igb_initialize_receive_units(adapter);
1351 /* Enable VLAN support */
1352 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1353 igb_setup_vlan_hw_support(adapter);
1355 /* Don't lose promiscuous settings */
1356 igb_set_promisc(adapter);
1358 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1359 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1361 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1362 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1364 if (adapter->msix > 1) /* Set up queue routing */
1365 igb_configure_queues(adapter);
1367 /* this clears any pending interrupts */
1368 E1000_READ_REG(&adapter->hw, E1000_ICR);
1369 #ifdef DEVICE_POLLING
1371 * Only enable interrupts if we are not polling, make sure
1372 * they are off otherwise.
1374 if (ifp->if_capenable & IFCAP_POLLING)
1375 igb_disable_intr(adapter);
1377 #endif /* DEVICE_POLLING */
1379 igb_enable_intr(adapter);
1380 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1383 /* Set Energy Efficient Ethernet */
1384 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
1385 if (adapter->hw.mac.type == e1000_i354)
1386 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
1388 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
1395 struct adapter *adapter = arg;
1397 IGB_CORE_LOCK(adapter);
1398 igb_init_locked(adapter);
1399 IGB_CORE_UNLOCK(adapter);
1404 igb_handle_que(void *context, int pending)
1406 struct igb_queue *que = context;
1407 struct adapter *adapter = que->adapter;
1408 struct tx_ring *txr = que->txr;
1409 struct ifnet *ifp = adapter->ifp;
1411 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1414 more = igb_rxeof(que, adapter->rx_process_limit, NULL);
1418 #ifndef IGB_LEGACY_TX
1419 /* Process the stack queue only if not depleted */
1420 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1421 !drbr_empty(ifp, txr->br))
1422 igb_mq_start_locked(ifp, txr);
1424 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1425 igb_start_locked(txr, ifp);
1428 /* Do we need another? */
1430 taskqueue_enqueue(que->tq, &que->que_task);
1435 #ifdef DEVICE_POLLING
1436 if (ifp->if_capenable & IFCAP_POLLING)
1439 /* Reenable this interrupt */
1441 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1443 igb_enable_intr(adapter);
1446 /* Deal with link in a sleepable context */
1448 igb_handle_link(void *context, int pending)
1450 struct adapter *adapter = context;
1452 IGB_CORE_LOCK(adapter);
1453 igb_handle_link_locked(adapter);
1454 IGB_CORE_UNLOCK(adapter);
1458 igb_handle_link_locked(struct adapter *adapter)
1460 struct tx_ring *txr = adapter->tx_rings;
1461 struct ifnet *ifp = adapter->ifp;
1463 IGB_CORE_LOCK_ASSERT(adapter);
1464 adapter->hw.mac.get_link_status = 1;
1465 igb_update_link_status(adapter);
1466 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
1467 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1469 #ifndef IGB_LEGACY_TX
1470 /* Process the stack queue only if not depleted */
1471 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1472 !drbr_empty(ifp, txr->br))
1473 igb_mq_start_locked(ifp, txr);
1475 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1476 igb_start_locked(txr, ifp);
1483 /*********************************************************************
1485 * MSI/Legacy Deferred
1486 * Interrupt Service routine
1488 *********************************************************************/
1490 igb_irq_fast(void *arg)
1492 struct adapter *adapter = arg;
1493 struct igb_queue *que = adapter->queues;
1497 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1500 if (reg_icr == 0xffffffff)
1501 return FILTER_STRAY;
1503 /* Definitely not our interrupt. */
1505 return FILTER_STRAY;
1507 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1508 return FILTER_STRAY;
1511 * Mask interrupts until the taskqueue is finished running. This is
1512 * cheap, just assume that it is needed. This also works around the
1513 * MSI message reordering errata on certain systems.
1515 igb_disable_intr(adapter);
1516 taskqueue_enqueue(que->tq, &que->que_task);
1518 /* Link status change */
1519 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1520 taskqueue_enqueue(que->tq, &adapter->link_task);
1522 if (reg_icr & E1000_ICR_RXO)
1523 adapter->rx_overruns++;
1524 return FILTER_HANDLED;
1527 #ifdef DEVICE_POLLING
1528 #if __FreeBSD_version >= 800000
1529 #define POLL_RETURN_COUNT(a) (a)
1532 #define POLL_RETURN_COUNT(a)
1535 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1537 struct adapter *adapter = ifp->if_softc;
1538 struct igb_queue *que;
1539 struct tx_ring *txr;
1540 u32 reg_icr, rx_done = 0;
1541 u32 loop = IGB_MAX_LOOP;
1544 IGB_CORE_LOCK(adapter);
1545 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1546 IGB_CORE_UNLOCK(adapter);
1547 return POLL_RETURN_COUNT(rx_done);
1550 if (cmd == POLL_AND_CHECK_STATUS) {
1551 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1552 /* Link status change */
1553 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1554 igb_handle_link_locked(adapter);
1556 if (reg_icr & E1000_ICR_RXO)
1557 adapter->rx_overruns++;
1559 IGB_CORE_UNLOCK(adapter);
1561 for (int i = 0; i < adapter->num_queues; i++) {
1562 que = &adapter->queues[i];
1565 igb_rxeof(que, count, &rx_done);
1569 more = igb_txeof(txr);
1570 } while (loop-- && more);
1571 #ifndef IGB_LEGACY_TX
1572 if (!drbr_empty(ifp, txr->br))
1573 igb_mq_start_locked(ifp, txr);
1575 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1576 igb_start_locked(txr, ifp);
1581 return POLL_RETURN_COUNT(rx_done);
1583 #endif /* DEVICE_POLLING */
1585 /*********************************************************************
1587 * MSIX Que Interrupt Service routine
1589 **********************************************************************/
1591 igb_msix_que(void *arg)
1593 struct igb_queue *que = arg;
1594 struct adapter *adapter = que->adapter;
1595 struct ifnet *ifp = adapter->ifp;
1596 struct tx_ring *txr = que->txr;
1597 struct rx_ring *rxr = que->rxr;
1601 /* Ignore spurious interrupts */
1602 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1605 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1610 #ifndef IGB_LEGACY_TX
1611 /* Process the stack queue only if not depleted */
1612 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1613 !drbr_empty(ifp, txr->br))
1614 igb_mq_start_locked(ifp, txr);
1616 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1617 igb_start_locked(txr, ifp);
1621 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
1623 if (adapter->enable_aim == FALSE)
1626 ** Do Adaptive Interrupt Moderation:
1627 ** - Write out last calculated setting
1628 ** - Calculate based on average size over
1629 ** the last interval.
1631 if (que->eitr_setting)
1632 E1000_WRITE_REG(&adapter->hw,
1633 E1000_EITR(que->msix), que->eitr_setting);
1635 que->eitr_setting = 0;
1637 /* Idle, do nothing */
1638 if ((txr->bytes == 0) && (rxr->bytes == 0))
1641 /* Used half Default if sub-gig */
1642 if (adapter->link_speed != 1000)
1643 newitr = IGB_DEFAULT_ITR / 2;
1645 if ((txr->bytes) && (txr->packets))
1646 newitr = txr->bytes/txr->packets;
1647 if ((rxr->bytes) && (rxr->packets))
1648 newitr = max(newitr,
1649 (rxr->bytes / rxr->packets));
1650 newitr += 24; /* account for hardware frame, crc */
1651 /* set an upper boundary */
1652 newitr = min(newitr, 3000);
1653 /* Be nice to the mid range */
1654 if ((newitr > 300) && (newitr < 1200))
1655 newitr = (newitr / 3);
1657 newitr = (newitr / 2);
1659 newitr &= 0x7FFC; /* Mask invalid bits */
1660 if (adapter->hw.mac.type == e1000_82575)
1661 newitr |= newitr << 16;
1663 newitr |= E1000_EITR_CNT_IGNR;
1665 /* save for next interrupt */
1666 que->eitr_setting = newitr;
1675 /* Schedule a clean task if needed*/
1677 taskqueue_enqueue(que->tq, &que->que_task);
1679 /* Reenable this interrupt */
1680 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1685 /*********************************************************************
1687 * MSIX Link Interrupt Service routine
1689 **********************************************************************/
1692 igb_msix_link(void *arg)
1694 struct adapter *adapter = arg;
1697 ++adapter->link_irq;
1698 icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1699 if (!(icr & E1000_ICR_LSC))
1701 igb_handle_link(adapter, 0);
1705 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1706 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1711 /*********************************************************************
1713 * Media Ioctl callback
1715 * This routine is called whenever the user queries the status of
1716 * the interface using ifconfig.
1718 **********************************************************************/
1720 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1722 struct adapter *adapter = ifp->if_softc;
1724 INIT_DEBUGOUT("igb_media_status: begin");
1726 IGB_CORE_LOCK(adapter);
1727 igb_update_link_status(adapter);
1729 ifmr->ifm_status = IFM_AVALID;
1730 ifmr->ifm_active = IFM_ETHER;
1732 if (!adapter->link_active) {
1733 IGB_CORE_UNLOCK(adapter);
1737 ifmr->ifm_status |= IFM_ACTIVE;
1739 switch (adapter->link_speed) {
1741 ifmr->ifm_active |= IFM_10_T;
1745 ** Support for 100Mb SFP - these are Fiber
1746 ** but the media type appears as serdes
1748 if (adapter->hw.phy.media_type ==
1749 e1000_media_type_internal_serdes)
1750 ifmr->ifm_active |= IFM_100_FX;
1752 ifmr->ifm_active |= IFM_100_TX;
1755 ifmr->ifm_active |= IFM_1000_T;
1758 ifmr->ifm_active |= IFM_2500_SX;
1762 if (adapter->link_duplex == FULL_DUPLEX)
1763 ifmr->ifm_active |= IFM_FDX;
1765 ifmr->ifm_active |= IFM_HDX;
1767 IGB_CORE_UNLOCK(adapter);
1770 /*********************************************************************
1772 * Media Ioctl callback
1774 * This routine is called when the user changes speed/duplex using
1775 * media/mediopt option with ifconfig.
1777 **********************************************************************/
1779 igb_media_change(struct ifnet *ifp)
1781 struct adapter *adapter = ifp->if_softc;
1782 struct ifmedia *ifm = &adapter->media;
1784 INIT_DEBUGOUT("igb_media_change: begin");
1786 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1789 IGB_CORE_LOCK(adapter);
1790 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1792 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1793 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1798 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1799 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1802 adapter->hw.mac.autoneg = FALSE;
1803 adapter->hw.phy.autoneg_advertised = 0;
1804 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1805 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1807 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1810 adapter->hw.mac.autoneg = FALSE;
1811 adapter->hw.phy.autoneg_advertised = 0;
1812 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1813 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1815 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1818 device_printf(adapter->dev, "Unsupported media type\n");
1821 igb_init_locked(adapter);
1822 IGB_CORE_UNLOCK(adapter);
1828 /*********************************************************************
1830 * This routine maps the mbufs to Advanced TX descriptors.
1832 **********************************************************************/
1834 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1836 struct adapter *adapter = txr->adapter;
1837 u32 olinfo_status = 0, cmd_type_len;
1838 int i, j, error, nsegs;
1841 struct mbuf *m_head;
1842 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1844 struct igb_tx_buf *txbuf;
1845 union e1000_adv_tx_desc *txd = NULL;
1849 /* Basic descriptor defines */
1850 cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
1851 E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
1853 if (m_head->m_flags & M_VLANTAG)
1854 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1857 * Important to capture the first descriptor
1858 * used because it will contain the index of
1859 * the one we tell the hardware to report back
1861 first = txr->next_avail_desc;
1862 txbuf = &txr->tx_buffers[first];
1866 * Map the packet for DMA.
1869 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1870 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1872 if (__predict_false(error)) {
1877 /* Try it again? - one try */
1878 if (remap == TRUE) {
1880 m = m_collapse(*m_headp, M_NOWAIT,
1883 adapter->mbuf_defrag_failed++;
1893 txr->no_tx_dma_setup++;
1900 /* Make certain there are enough descriptors */
1901 if (txr->tx_avail < (nsegs + 2)) {
1902 txr->no_desc_avail++;
1903 bus_dmamap_unload(txr->txtag, map);
1909 ** Set up the appropriate offload context
1910 ** this will consume the first descriptor
1912 error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1913 if (__predict_false(error)) {
1919 /* 82575 needs the queue index added */
1920 if (adapter->hw.mac.type == e1000_82575)
1921 olinfo_status |= txr->me << 4;
1923 i = txr->next_avail_desc;
1924 for (j = 0; j < nsegs; j++) {
1928 txbuf = &txr->tx_buffers[i];
1929 txd = &txr->tx_base[i];
1930 seglen = segs[j].ds_len;
1931 segaddr = htole64(segs[j].ds_addr);
1933 txd->read.buffer_addr = segaddr;
1934 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
1935 cmd_type_len | seglen);
1936 txd->read.olinfo_status = htole32(olinfo_status);
1938 if (++i == txr->num_desc)
1942 txd->read.cmd_type_len |=
1943 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1944 txr->tx_avail -= nsegs;
1945 txr->next_avail_desc = i;
1947 txbuf->m_head = m_head;
1949 ** Here we swap the map so the last descriptor,
1950 ** which gets the completion interrupt has the
1951 ** real map, and the first descriptor gets the
1952 ** unused map from this descriptor.
1954 txr->tx_buffers[first].map = txbuf->map;
1956 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1958 /* Set the EOP descriptor that will be marked done */
1959 txbuf = &txr->tx_buffers[first];
1962 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1963 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1965 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1966 * hardware that this frame is available to transmit.
1968 ++txr->total_packets;
1969 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1974 igb_set_promisc(struct adapter *adapter)
1976 struct ifnet *ifp = adapter->ifp;
1977 struct e1000_hw *hw = &adapter->hw;
1980 if (adapter->vf_ifp) {
1981 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
1985 reg = E1000_READ_REG(hw, E1000_RCTL);
1986 if (ifp->if_flags & IFF_PROMISC) {
1987 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1988 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1989 } else if (ifp->if_flags & IFF_ALLMULTI) {
1990 reg |= E1000_RCTL_MPE;
1991 reg &= ~E1000_RCTL_UPE;
1992 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1997 igb_disable_promisc(struct adapter *adapter)
1999 struct e1000_hw *hw = &adapter->hw;
2000 struct ifnet *ifp = adapter->ifp;
2004 if (adapter->vf_ifp) {
2005 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
2008 reg = E1000_READ_REG(hw, E1000_RCTL);
2009 reg &= (~E1000_RCTL_UPE);
2010 if (ifp->if_flags & IFF_ALLMULTI)
2011 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2013 struct ifmultiaddr *ifma;
2014 #if __FreeBSD_version < 800000
2017 if_maddr_rlock(ifp);
2019 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2020 if (ifma->ifma_addr->sa_family != AF_LINK)
2022 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2026 #if __FreeBSD_version < 800000
2027 IF_ADDR_UNLOCK(ifp);
2029 if_maddr_runlock(ifp);
2032 /* Don't disable if in MAX groups */
2033 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2034 reg &= (~E1000_RCTL_MPE);
2035 E1000_WRITE_REG(hw, E1000_RCTL, reg);
2039 /*********************************************************************
2042 * This routine is called whenever multicast address list is updated.
2044 **********************************************************************/
2047 igb_set_multi(struct adapter *adapter)
2049 struct ifnet *ifp = adapter->ifp;
2050 struct ifmultiaddr *ifma;
2056 IOCTL_DEBUGOUT("igb_set_multi: begin");
2059 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
2060 MAX_NUM_MULTICAST_ADDRESSES);
2062 #if __FreeBSD_version < 800000
2065 if_maddr_rlock(ifp);
2067 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2068 if (ifma->ifma_addr->sa_family != AF_LINK)
2071 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2074 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2075 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2078 #if __FreeBSD_version < 800000
2079 IF_ADDR_UNLOCK(ifp);
2081 if_maddr_runlock(ifp);
2084 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2085 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2086 reg_rctl |= E1000_RCTL_MPE;
2087 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2089 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2093 /*********************************************************************
2095 * This routine checks for link status,
2096 * updates statistics, and does the watchdog.
2098 **********************************************************************/
2101 igb_local_timer(void *arg)
2103 struct adapter *adapter = arg;
2104 device_t dev = adapter->dev;
2105 struct ifnet *ifp = adapter->ifp;
2106 struct tx_ring *txr = adapter->tx_rings;
2107 struct igb_queue *que = adapter->queues;
2108 int hung = 0, busy = 0;
2111 IGB_CORE_LOCK_ASSERT(adapter);
2113 igb_update_link_status(adapter);
2114 igb_update_stats_counters(adapter);
2117 ** Check the TX queues status
2118 ** - central locked handling of OACTIVE
2119 ** - watchdog only if all queues show hung
2121 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2122 if ((txr->queue_status & IGB_QUEUE_HUNG) &&
2123 (adapter->pause_frames == 0))
2125 if (txr->queue_status & IGB_QUEUE_DEPLETED)
2127 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
2128 taskqueue_enqueue(que->tq, &que->que_task);
2130 if (hung == adapter->num_queues)
2132 if (busy == adapter->num_queues)
2133 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2134 else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
2135 (busy < adapter->num_queues))
2136 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2138 adapter->pause_frames = 0;
2139 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
2140 #ifndef DEVICE_POLLING
2141 /* Schedule all queue interrupts - deadlock protection */
2142 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
2147 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2148 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2149 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2150 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2151 device_printf(dev,"TX(%d) desc avail = %d,"
2152 "Next TX to Clean = %d\n",
2153 txr->me, txr->tx_avail, txr->next_to_clean);
2154 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2155 adapter->watchdog_events++;
2156 igb_init_locked(adapter);
2160 igb_update_link_status(struct adapter *adapter)
2162 struct e1000_hw *hw = &adapter->hw;
2163 struct e1000_fc_info *fc = &hw->fc;
2164 struct ifnet *ifp = adapter->ifp;
2165 device_t dev = adapter->dev;
2166 struct tx_ring *txr = adapter->tx_rings;
2167 u32 link_check, thstat, ctrl;
2168 char *flowctl = NULL;
2170 link_check = thstat = ctrl = 0;
2172 /* Get the cached link value or read for real */
2173 switch (hw->phy.media_type) {
2174 case e1000_media_type_copper:
2175 if (hw->mac.get_link_status) {
2176 /* Do the work to read phy */
2177 e1000_check_for_link(hw);
2178 link_check = !hw->mac.get_link_status;
2182 case e1000_media_type_fiber:
2183 e1000_check_for_link(hw);
2184 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2187 case e1000_media_type_internal_serdes:
2188 e1000_check_for_link(hw);
2189 link_check = adapter->hw.mac.serdes_has_link;
2191 /* VF device is type_unknown */
2192 case e1000_media_type_unknown:
2193 e1000_check_for_link(hw);
2194 link_check = !hw->mac.get_link_status;
2200 /* Check for thermal downshift or shutdown */
2201 if (hw->mac.type == e1000_i350) {
2202 thstat = E1000_READ_REG(hw, E1000_THSTAT);
2203 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
2206 /* Get the flow control for display */
2207 switch (fc->current_mode) {
2208 case e1000_fc_rx_pause:
2211 case e1000_fc_tx_pause:
2223 /* Now we check if a transition has happened */
2224 if (link_check && (adapter->link_active == 0)) {
2225 e1000_get_speed_and_duplex(&adapter->hw,
2226 &adapter->link_speed, &adapter->link_duplex);
2228 device_printf(dev, "Link is up %d Mbps %s,"
2229 " Flow Control: %s\n",
2230 adapter->link_speed,
2231 ((adapter->link_duplex == FULL_DUPLEX) ?
2232 "Full Duplex" : "Half Duplex"), flowctl);
2233 adapter->link_active = 1;
2234 ifp->if_baudrate = adapter->link_speed * 1000000;
2235 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2236 (thstat & E1000_THSTAT_LINK_THROTTLE))
2237 device_printf(dev, "Link: thermal downshift\n");
2238 /* Delay Link Up for Phy update */
2239 if (((hw->mac.type == e1000_i210) ||
2240 (hw->mac.type == e1000_i211)) &&
2241 (hw->phy.id == I210_I_PHY_ID))
2242 msec_delay(I210_LINK_DELAY);
2243 /* Reset if the media type changed. */
2244 if (hw->dev_spec._82575.media_changed) {
2245 hw->dev_spec._82575.media_changed = false;
2246 adapter->flags |= IGB_MEDIA_RESET;
2249 /* This can sleep */
2250 if_link_state_change(ifp, LINK_STATE_UP);
2251 } else if (!link_check && (adapter->link_active == 1)) {
2252 ifp->if_baudrate = adapter->link_speed = 0;
2253 adapter->link_duplex = 0;
2255 device_printf(dev, "Link is Down\n");
2256 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2257 (thstat & E1000_THSTAT_PWR_DOWN))
2258 device_printf(dev, "Link: thermal shutdown\n");
2259 adapter->link_active = 0;
2260 /* This can sleep */
2261 if_link_state_change(ifp, LINK_STATE_DOWN);
2262 /* Reset queue state */
2263 for (int i = 0; i < adapter->num_queues; i++, txr++)
2264 txr->queue_status = IGB_QUEUE_IDLE;
2268 /*********************************************************************
2270 * This routine disables all traffic on the adapter by issuing a
2271 * global reset on the MAC and deallocates TX/RX buffers.
2273 **********************************************************************/
2278 struct adapter *adapter = arg;
2279 struct ifnet *ifp = adapter->ifp;
2280 struct tx_ring *txr = adapter->tx_rings;
2282 IGB_CORE_LOCK_ASSERT(adapter);
2284 INIT_DEBUGOUT("igb_stop: begin");
2286 igb_disable_intr(adapter);
2288 callout_stop(&adapter->timer);
2290 /* Tell the stack that the interface is no longer active */
2291 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2292 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2294 /* Disarm watchdog timer. */
2295 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2297 txr->queue_status = IGB_QUEUE_IDLE;
2301 e1000_reset_hw(&adapter->hw);
2302 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2304 e1000_led_off(&adapter->hw);
2305 e1000_cleanup_led(&adapter->hw);
2309 /*********************************************************************
2311 * Determine hardware revision.
2313 **********************************************************************/
2315 igb_identify_hardware(struct adapter *adapter)
2317 device_t dev = adapter->dev;
2319 /* Make sure our PCI config space has the necessary stuff set */
2320 pci_enable_busmaster(dev);
2321 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2323 /* Save off the information about this board */
2324 adapter->hw.vendor_id = pci_get_vendor(dev);
2325 adapter->hw.device_id = pci_get_device(dev);
2326 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2327 adapter->hw.subsystem_vendor_id =
2328 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2329 adapter->hw.subsystem_device_id =
2330 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2332 /* Set MAC type early for PCI setup */
2333 e1000_set_mac_type(&adapter->hw);
2335 /* Are we a VF device? */
2336 if ((adapter->hw.mac.type == e1000_vfadapt) ||
2337 (adapter->hw.mac.type == e1000_vfadapt_i350))
2338 adapter->vf_ifp = 1;
2340 adapter->vf_ifp = 0;
2344 igb_allocate_pci_resources(struct adapter *adapter)
2346 device_t dev = adapter->dev;
2350 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2352 if (adapter->pci_mem == NULL) {
2353 device_printf(dev, "Unable to allocate bus resource: memory\n");
2356 adapter->osdep.mem_bus_space_tag =
2357 rman_get_bustag(adapter->pci_mem);
2358 adapter->osdep.mem_bus_space_handle =
2359 rman_get_bushandle(adapter->pci_mem);
2360 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2362 adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2364 /* This will setup either MSI/X or MSI */
2365 adapter->msix = igb_setup_msix(adapter);
2366 adapter->hw.back = &adapter->osdep;
2371 /*********************************************************************
2373 * Setup the Legacy or MSI Interrupt handler
2375 **********************************************************************/
2377 igb_allocate_legacy(struct adapter *adapter)
2379 device_t dev = adapter->dev;
2380 struct igb_queue *que = adapter->queues;
2381 #ifndef IGB_LEGACY_TX
2382 struct tx_ring *txr = adapter->tx_rings;
2386 /* Turn off all interrupts */
2387 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2390 if (adapter->msix == 1)
2393 /* We allocate a single interrupt resource */
2394 adapter->res = bus_alloc_resource_any(dev,
2395 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2396 if (adapter->res == NULL) {
2397 device_printf(dev, "Unable to allocate bus resource: "
2402 #ifndef IGB_LEGACY_TX
2403 TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
2407 * Try allocating a fast interrupt and the associated deferred
2408 * processing contexts.
2410 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2411 /* Make tasklet for deferred link handling */
2412 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2413 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2414 taskqueue_thread_enqueue, &que->tq);
2415 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
2416 device_get_nameunit(adapter->dev));
2417 if ((error = bus_setup_intr(dev, adapter->res,
2418 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2419 adapter, &adapter->tag)) != 0) {
2420 device_printf(dev, "Failed to register fast interrupt "
2421 "handler: %d\n", error);
2422 taskqueue_free(que->tq);
2431 /*********************************************************************
2433 * Setup the MSIX Queue Interrupt handlers:
2435 **********************************************************************/
2437 igb_allocate_msix(struct adapter *adapter)
2439 device_t dev = adapter->dev;
2440 struct igb_queue *que = adapter->queues;
2441 int error, rid, vector = 0;
2447 /* Be sure to start with all interrupts disabled */
2448 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
2449 E1000_WRITE_FLUSH(&adapter->hw);
2453 * If we're doing RSS, the number of queues needs to
2454 * match the number of RSS buckets that are configured.
2456 * + If there's more queues than RSS buckets, we'll end
2457 * up with queues that get no traffic.
2459 * + If there's more RSS buckets than queues, we'll end
2460 * up having multiple RSS buckets map to the same queue,
2461 * so there'll be some contention.
2463 if (adapter->num_queues != rss_getnumbuckets()) {
2465 "%s: number of queues (%d) != number of RSS buckets (%d)"
2466 "; performance will be impacted.\n",
2468 adapter->num_queues,
2469 rss_getnumbuckets());
2473 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2475 que->res = bus_alloc_resource_any(dev,
2476 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2477 if (que->res == NULL) {
2479 "Unable to allocate bus resource: "
2480 "MSIX Queue Interrupt\n");
2483 error = bus_setup_intr(dev, que->res,
2484 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2485 igb_msix_que, que, &que->tag);
2488 device_printf(dev, "Failed to register Queue handler");
2491 #if __FreeBSD_version >= 800504
2492 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2495 if (adapter->hw.mac.type == e1000_82575)
2496 que->eims = E1000_EICR_TX_QUEUE0 << i;
2498 que->eims = 1 << vector;
2502 * The queue ID is used as the RSS layer bucket ID.
2503 * We look up the queue ID -> RSS CPU ID and select
2506 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2509 * Bind the msix vector, and thus the
2510 * rings to the corresponding cpu.
2512 * This just happens to match the default RSS round-robin
2513 * bucket -> queue -> CPU allocation.
2515 if (adapter->num_queues > 1) {
2516 if (igb_last_bind_cpu < 0)
2517 igb_last_bind_cpu = CPU_FIRST();
2518 cpu_id = igb_last_bind_cpu;
2522 if (adapter->num_queues > 1) {
2523 bus_bind_intr(dev, que->res, cpu_id);
2526 "Bound queue %d to RSS bucket %d\n",
2530 "Bound queue %d to cpu %d\n",
2535 #ifndef IGB_LEGACY_TX
2536 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
2539 /* Make tasklet for deferred handling */
2540 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2541 que->tq = taskqueue_create("igb_que", M_NOWAIT,
2542 taskqueue_thread_enqueue, &que->tq);
2543 if (adapter->num_queues > 1) {
2545 * Only pin the taskqueue thread to a CPU if
2548 * This again just happens to match the default RSS
2549 * round-robin bucket -> queue -> CPU allocation.
2552 CPU_SETOF(cpu_id, &cpu_mask);
2553 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2555 "%s que (bucket %d)",
2556 device_get_nameunit(adapter->dev),
2559 taskqueue_start_threads(&que->tq, 1, PI_NET,
2561 device_get_nameunit(adapter->dev),
2565 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2566 device_get_nameunit(adapter->dev));
2569 /* Finally update the last bound CPU id */
2570 if (adapter->num_queues > 1)
2571 igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
2576 adapter->res = bus_alloc_resource_any(dev,
2577 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2578 if (adapter->res == NULL) {
2580 "Unable to allocate bus resource: "
2581 "MSIX Link Interrupt\n");
2584 if ((error = bus_setup_intr(dev, adapter->res,
2585 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2586 igb_msix_link, adapter, &adapter->tag)) != 0) {
2587 device_printf(dev, "Failed to register Link handler");
2590 #if __FreeBSD_version >= 800504
2591 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2593 adapter->linkvec = vector;
2600 igb_configure_queues(struct adapter *adapter)
2602 struct e1000_hw *hw = &adapter->hw;
2603 struct igb_queue *que;
2604 u32 tmp, ivar = 0, newitr = 0;
2606 /* First turn on RSS capability */
2607 if (adapter->hw.mac.type != e1000_82575)
2608 E1000_WRITE_REG(hw, E1000_GPIE,
2609 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2610 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2613 switch (adapter->hw.mac.type) {
2620 case e1000_vfadapt_i350:
2622 for (int i = 0; i < adapter->num_queues; i++) {
2624 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2625 que = &adapter->queues[i];
2628 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2631 ivar |= que->msix | E1000_IVAR_VALID;
2633 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2636 for (int i = 0; i < adapter->num_queues; i++) {
2638 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2639 que = &adapter->queues[i];
2642 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2645 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2647 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2648 adapter->que_mask |= que->eims;
2651 /* And for the link interrupt */
2652 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2653 adapter->link_mask = 1 << adapter->linkvec;
2654 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2658 for (int i = 0; i < adapter->num_queues; i++) {
2659 u32 index = i & 0x7; /* Each IVAR has two entries */
2660 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2661 que = &adapter->queues[i];
2664 ivar |= que->msix | E1000_IVAR_VALID;
2667 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2669 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2670 adapter->que_mask |= que->eims;
2673 for (int i = 0; i < adapter->num_queues; i++) {
2674 u32 index = i & 0x7; /* Each IVAR has two entries */
2675 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2676 que = &adapter->queues[i];
2679 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2682 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2684 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2685 adapter->que_mask |= que->eims;
2688 /* And for the link interrupt */
2689 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2690 adapter->link_mask = 1 << adapter->linkvec;
2691 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2695 /* enable MSI-X support*/
2696 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2697 tmp |= E1000_CTRL_EXT_PBA_CLR;
2698 /* Auto-Mask interrupts upon ICR read. */
2699 tmp |= E1000_CTRL_EXT_EIAME;
2700 tmp |= E1000_CTRL_EXT_IRCA;
2701 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2704 for (int i = 0; i < adapter->num_queues; i++) {
2705 que = &adapter->queues[i];
2706 tmp = E1000_EICR_RX_QUEUE0 << i;
2707 tmp |= E1000_EICR_TX_QUEUE0 << i;
2709 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2711 adapter->que_mask |= que->eims;
2715 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2717 adapter->link_mask |= E1000_EIMS_OTHER;
2722 /* Set the starting interrupt rate */
2723 if (igb_max_interrupt_rate > 0)
2724 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
2726 if (hw->mac.type == e1000_82575)
2727 newitr |= newitr << 16;
2729 newitr |= E1000_EITR_CNT_IGNR;
2731 for (int i = 0; i < adapter->num_queues; i++) {
2732 que = &adapter->queues[i];
2733 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2741 igb_free_pci_resources(struct adapter *adapter)
2743 struct igb_queue *que = adapter->queues;
2744 device_t dev = adapter->dev;
2748 ** There is a slight possibility of a failure mode
2749 ** in attach that will result in entering this function
2750 ** before interrupt resources have been initialized, and
2751 ** in that case we do not want to execute the loops below
2752 ** We can detect this reliably by the state of the adapter
2755 if (adapter->res == NULL)
2759 * First release all the interrupt resources:
2761 for (int i = 0; i < adapter->num_queues; i++, que++) {
2762 rid = que->msix + 1;
2763 if (que->tag != NULL) {
2764 bus_teardown_intr(dev, que->res, que->tag);
2767 if (que->res != NULL)
2768 bus_release_resource(dev,
2769 SYS_RES_IRQ, rid, que->res);
2772 /* Clean the Legacy or Link interrupt last */
2773 if (adapter->linkvec) /* we are doing MSIX */
2774 rid = adapter->linkvec + 1;
2776 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2778 que = adapter->queues;
2779 if (adapter->tag != NULL) {
2780 taskqueue_drain(que->tq, &adapter->link_task);
2781 bus_teardown_intr(dev, adapter->res, adapter->tag);
2782 adapter->tag = NULL;
2784 if (adapter->res != NULL)
2785 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2787 for (int i = 0; i < adapter->num_queues; i++, que++) {
2788 if (que->tq != NULL) {
2789 #ifndef IGB_LEGACY_TX
2790 taskqueue_drain(que->tq, &que->txr->txq_task);
2792 taskqueue_drain(que->tq, &que->que_task);
2793 taskqueue_free(que->tq);
2798 pci_release_msi(dev);
2800 if (adapter->msix_mem != NULL)
2801 bus_release_resource(dev, SYS_RES_MEMORY,
2802 adapter->memrid, adapter->msix_mem);
2804 if (adapter->pci_mem != NULL)
2805 bus_release_resource(dev, SYS_RES_MEMORY,
2806 PCIR_BAR(0), adapter->pci_mem);
2811 * Setup Either MSI/X or MSI
2814 igb_setup_msix(struct adapter *adapter)
2816 device_t dev = adapter->dev;
2817 int bar, want, queues, msgs, maxqueues;
2819 /* tuneable override */
2820 if (igb_enable_msix == 0)
2823 /* First try MSI/X */
2824 msgs = pci_msix_count(dev);
2828 ** Some new devices, as with ixgbe, now may
2829 ** use a different BAR, so we need to keep
2830 ** track of which is used.
2832 adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
2833 bar = pci_read_config(dev, adapter->memrid, 4);
2834 if (bar == 0) /* use next bar */
2835 adapter->memrid += 4;
2836 adapter->msix_mem = bus_alloc_resource_any(dev,
2837 SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
2838 if (adapter->msix_mem == NULL) {
2839 /* May not be enabled */
2840 device_printf(adapter->dev,
2841 "Unable to map MSIX table \n");
2845 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2847 /* Override via tuneable */
2848 if (igb_num_queues != 0)
2849 queues = igb_num_queues;
2852 /* If we're doing RSS, clamp at the number of RSS buckets */
2853 if (queues > rss_getnumbuckets())
2854 queues = rss_getnumbuckets();
2858 /* Sanity check based on HW */
2859 switch (adapter->hw.mac.type) {
2875 default: /* VF interfaces */
2880 /* Final clamp on the actual hardware capability */
2881 if (queues > maxqueues)
2885 ** One vector (RX/TX pair) per queue
2886 ** plus an additional for Link interrupt
2892 device_printf(adapter->dev,
2893 "MSIX Configuration Problem, "
2894 "%d vectors configured, but %d queues wanted!\n",
2898 if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2899 device_printf(adapter->dev,
2900 "Using MSIX interrupts with %d vectors\n", msgs);
2901 adapter->num_queues = queues;
2905 ** If MSIX alloc failed or provided us with
2906 ** less than needed, free and fall through to MSI
2908 pci_release_msi(dev);
2911 if (adapter->msix_mem != NULL) {
2912 bus_release_resource(dev, SYS_RES_MEMORY,
2913 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2914 adapter->msix_mem = NULL;
2917 if (pci_alloc_msi(dev, &msgs) == 0) {
2918 device_printf(adapter->dev," Using an MSI interrupt\n");
2921 device_printf(adapter->dev," Using a Legacy interrupt\n");
2925 /*********************************************************************
2927 * Initialize the DMA Coalescing feature
2929 **********************************************************************/
2931 igb_init_dmac(struct adapter *adapter, u32 pba)
2933 device_t dev = adapter->dev;
2934 struct e1000_hw *hw = &adapter->hw;
2935 u32 dmac, reg = ~E1000_DMACR_DMAC_EN;
2938 if (hw->mac.type == e1000_i211)
2941 if (hw->mac.type > e1000_82580) {
2943 if (adapter->dmac == 0) { /* Disabling it */
2944 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2947 device_printf(dev, "DMA Coalescing enabled\n");
2949 /* Set starting threshold */
2950 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
2952 hwm = 64 * pba - adapter->max_frame_size / 16;
2953 if (hwm < 64 * (pba - 6))
2954 hwm = 64 * (pba - 6);
2955 reg = E1000_READ_REG(hw, E1000_FCRTC);
2956 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
2957 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
2958 & E1000_FCRTC_RTH_COAL_MASK);
2959 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
2962 dmac = pba - adapter->max_frame_size / 512;
2963 if (dmac < pba - 10)
2965 reg = E1000_READ_REG(hw, E1000_DMACR);
2966 reg &= ~E1000_DMACR_DMACTHR_MASK;
2967 reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
2968 & E1000_DMACR_DMACTHR_MASK);
2970 /* transition to L0x or L1 if available..*/
2971 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
2973 /* Check if status is 2.5Gb backplane connection
2974 * before configuration of watchdog timer, which is
2975 * in msec values in 12.8usec intervals
2976 * watchdog timer= msec values in 32usec intervals
2977 * for non 2.5Gb connection
2979 if (hw->mac.type == e1000_i354) {
2980 int status = E1000_READ_REG(hw, E1000_STATUS);
2981 if ((status & E1000_STATUS_2P5_SKU) &&
2982 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2983 reg |= ((adapter->dmac * 5) >> 6);
2985 reg |= (adapter->dmac >> 5);
2987 reg |= (adapter->dmac >> 5);
2990 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2992 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
2994 /* Set the interval before transition */
2995 reg = E1000_READ_REG(hw, E1000_DMCTLX);
2996 if (hw->mac.type == e1000_i350)
2997 reg |= IGB_DMCTLX_DCFLUSH_DIS;
2999 ** in 2.5Gb connection, TTLX unit is 0.4 usec
3000 ** which is 0x4*2 = 0xA. But delay is still 4 usec
3002 if (hw->mac.type == e1000_i354) {
3003 int status = E1000_READ_REG(hw, E1000_STATUS);
3004 if ((status & E1000_STATUS_2P5_SKU) &&
3005 (!(status & E1000_STATUS_2P5_SKU_OVER)))
3013 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
3015 /* free space in tx packet buffer to wake from DMA coal */
3016 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
3017 (2 * adapter->max_frame_size)) >> 6);
3019 /* make low power state decision controlled by DMA coal */
3020 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
3021 reg &= ~E1000_PCIEMISC_LX_DECISION;
3022 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
3024 } else if (hw->mac.type == e1000_82580) {
3025 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
3026 E1000_WRITE_REG(hw, E1000_PCIEMISC,
3027 reg & ~E1000_PCIEMISC_LX_DECISION);
3028 E1000_WRITE_REG(hw, E1000_DMACR, 0);
3033 /*********************************************************************
3035 * Set up an fresh starting state
3037 **********************************************************************/
3039 igb_reset(struct adapter *adapter)
3041 device_t dev = adapter->dev;
3042 struct e1000_hw *hw = &adapter->hw;
3043 struct e1000_fc_info *fc = &hw->fc;
3044 struct ifnet *ifp = adapter->ifp;
3048 INIT_DEBUGOUT("igb_reset: begin");
3050 /* Let the firmware know the OS is in control */
3051 igb_get_hw_control(adapter);
3054 * Packet Buffer Allocation (PBA)
3055 * Writing PBA sets the receive portion of the buffer
3056 * the remainder is used for the transmit buffer.
3058 switch (hw->mac.type) {
3060 pba = E1000_PBA_32K;
3064 pba = E1000_READ_REG(hw, E1000_RXPBS);
3065 pba &= E1000_RXPBS_SIZE_MASK_82576;
3070 case e1000_vfadapt_i350:
3071 pba = E1000_READ_REG(hw, E1000_RXPBS);
3072 pba = e1000_rxpbs_adjust_82580(pba);
3076 pba = E1000_PBA_34K;
3081 /* Special needs in case of Jumbo frames */
3082 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
3083 u32 tx_space, min_tx, min_rx;
3084 pba = E1000_READ_REG(hw, E1000_PBA);
3085 tx_space = pba >> 16;
3087 min_tx = (adapter->max_frame_size +
3088 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
3089 min_tx = roundup2(min_tx, 1024);
3091 min_rx = adapter->max_frame_size;
3092 min_rx = roundup2(min_rx, 1024);
3094 if (tx_space < min_tx &&
3095 ((min_tx - tx_space) < pba)) {
3096 pba = pba - (min_tx - tx_space);
3098 * if short on rx space, rx wins
3099 * and must trump tx adjustment
3104 E1000_WRITE_REG(hw, E1000_PBA, pba);
3107 INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
3110 * These parameters control the automatic generation (Tx) and
3111 * response (Rx) to Ethernet PAUSE frames.
3112 * - High water mark should allow for at least two frames to be
3113 * received after sending an XOFF.
3114 * - Low water mark works best when it is very near the high water mark.
3115 * This allows the receiver to restart by sending XON when it has
3118 hwm = min(((pba << 10) * 9 / 10),
3119 ((pba << 10) - 2 * adapter->max_frame_size));
3121 if (hw->mac.type < e1000_82576) {
3122 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
3123 fc->low_water = fc->high_water - 8;
3125 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
3126 fc->low_water = fc->high_water - 16;
3129 fc->pause_time = IGB_FC_PAUSE_TIME;
3130 fc->send_xon = TRUE;
3132 fc->requested_mode = adapter->fc;
3134 fc->requested_mode = e1000_fc_default;
3136 /* Issue a global reset */
3138 E1000_WRITE_REG(hw, E1000_WUC, 0);
3140 /* Reset for AutoMediaDetect */
3141 if (adapter->flags & IGB_MEDIA_RESET) {
3142 e1000_setup_init_funcs(hw, TRUE);
3143 e1000_get_bus_info(hw);
3144 adapter->flags &= ~IGB_MEDIA_RESET;
3147 if (e1000_init_hw(hw) < 0)
3148 device_printf(dev, "Hardware Initialization Failed\n");
3150 /* Setup DMA Coalescing */
3151 igb_init_dmac(adapter, pba);
3153 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
3154 e1000_get_phy_info(hw);
3155 e1000_check_for_link(hw);
3159 /*********************************************************************
3161 * Setup networking device structure and register an interface.
3163 **********************************************************************/
3165 igb_setup_interface(device_t dev, struct adapter *adapter)
3169 INIT_DEBUGOUT("igb_setup_interface: begin");
3171 ifp = adapter->ifp = if_alloc(IFT_ETHER);
3173 device_printf(dev, "can not allocate ifnet structure\n");
3176 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3177 ifp->if_init = igb_init;
3178 ifp->if_softc = adapter;
3179 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3180 ifp->if_ioctl = igb_ioctl;
3181 ifp->if_get_counter = igb_get_counter;
3183 /* TSO parameters */
3184 ifp->if_hw_tsomax = IP_MAXPACKET;
3185 ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER;
3186 ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE;
3188 #ifndef IGB_LEGACY_TX
3189 ifp->if_transmit = igb_mq_start;
3190 ifp->if_qflush = igb_qflush;
3192 ifp->if_start = igb_start;
3193 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3194 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3195 IFQ_SET_READY(&ifp->if_snd);
3198 ether_ifattach(ifp, adapter->hw.mac.addr);
3200 ifp->if_capabilities = ifp->if_capenable = 0;
3202 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3203 #if __FreeBSD_version >= 1000000
3204 ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
3206 ifp->if_capabilities |= IFCAP_TSO;
3207 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
3208 ifp->if_capenable = ifp->if_capabilities;
3210 /* Don't enable LRO by default */
3211 ifp->if_capabilities |= IFCAP_LRO;
3213 #ifdef DEVICE_POLLING
3214 ifp->if_capabilities |= IFCAP_POLLING;
3218 * Tell the upper layer(s) we
3219 * support full VLAN capability.
3221 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
3222 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
3225 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
3230 ** Don't turn this on by default, if vlans are
3231 ** created on another pseudo device (eg. lagg)
3232 ** then vlan events are not passed thru, breaking
3233 ** operation, but with HW FILTER off it works. If
3234 ** using vlans directly on the igb driver you can
3235 ** enable this and get full hardware tag filtering.
3237 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3240 * Specify the media types supported by this adapter and register
3241 * callbacks to update media and link information
3243 ifmedia_init(&adapter->media, IFM_IMASK,
3244 igb_media_change, igb_media_status);
3245 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3246 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3247 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
3249 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
3251 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3252 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3254 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3256 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3258 if (adapter->hw.phy.type != e1000_phy_ife) {
3259 ifmedia_add(&adapter->media,
3260 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3261 ifmedia_add(&adapter->media,
3262 IFM_ETHER | IFM_1000_T, 0, NULL);
3265 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3266 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3272 * Manage DMA'able memory.
3275 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3279 *(bus_addr_t *) arg = segs[0].ds_addr;
3283 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
3284 struct igb_dma_alloc *dma, int mapflags)
3288 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3289 IGB_DBA_ALIGN, 0, /* alignment, bounds */
3290 BUS_SPACE_MAXADDR, /* lowaddr */
3291 BUS_SPACE_MAXADDR, /* highaddr */
3292 NULL, NULL, /* filter, filterarg */
3295 size, /* maxsegsize */
3297 NULL, /* lockfunc */
3301 device_printf(adapter->dev,
3302 "%s: bus_dma_tag_create failed: %d\n",
3307 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3308 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3310 device_printf(adapter->dev,
3311 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3312 __func__, (uintmax_t)size, error);
3317 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3318 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3319 if (error || dma->dma_paddr == 0) {
3320 device_printf(adapter->dev,
3321 "%s: bus_dmamap_load failed: %d\n",
3329 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3331 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3332 bus_dma_tag_destroy(dma->dma_tag);
3334 dma->dma_tag = NULL;
3340 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
3342 if (dma->dma_tag == NULL)
3344 if (dma->dma_paddr != 0) {
3345 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3346 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3347 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3350 if (dma->dma_vaddr != NULL) {
3351 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3352 dma->dma_vaddr = NULL;
3354 bus_dma_tag_destroy(dma->dma_tag);
3355 dma->dma_tag = NULL;
3359 /*********************************************************************
3361 * Allocate memory for the transmit and receive rings, and then
3362 * the descriptors associated with each, called only once at attach.
3364 **********************************************************************/
3366 igb_allocate_queues(struct adapter *adapter)
3368 device_t dev = adapter->dev;
3369 struct igb_queue *que = NULL;
3370 struct tx_ring *txr = NULL;
3371 struct rx_ring *rxr = NULL;
3372 int rsize, tsize, error = E1000_SUCCESS;
3373 int txconf = 0, rxconf = 0;
3375 /* First allocate the top level queue structs */
3376 if (!(adapter->queues =
3377 (struct igb_queue *) malloc(sizeof(struct igb_queue) *
3378 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3379 device_printf(dev, "Unable to allocate queue memory\n");
3384 /* Next allocate the TX ring struct memory */
3385 if (!(adapter->tx_rings =
3386 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3387 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3388 device_printf(dev, "Unable to allocate TX ring memory\n");
3393 /* Now allocate the RX */
3394 if (!(adapter->rx_rings =
3395 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3396 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3397 device_printf(dev, "Unable to allocate RX ring memory\n");
3402 tsize = roundup2(adapter->num_tx_desc *
3403 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
3405 * Now set up the TX queues, txconf is needed to handle the
3406 * possibility that things fail midcourse and we need to
3407 * undo memory gracefully
3409 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3410 /* Set up some basics */
3411 txr = &adapter->tx_rings[i];
3412 txr->adapter = adapter;
3414 txr->num_desc = adapter->num_tx_desc;
3416 /* Initialize the TX lock */
3417 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3418 device_get_nameunit(dev), txr->me);
3419 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3421 if (igb_dma_malloc(adapter, tsize,
3422 &txr->txdma, BUS_DMA_NOWAIT)) {
3424 "Unable to allocate TX Descriptor memory\n");
3428 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
3429 bzero((void *)txr->tx_base, tsize);
3431 /* Now allocate transmit buffers for the ring */
3432 if (igb_allocate_transmit_buffers(txr)) {
3434 "Critical Failure setting up transmit buffers\n");
3438 #ifndef IGB_LEGACY_TX
3439 /* Allocate a buf ring */
3440 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
3441 M_WAITOK, &txr->tx_mtx);
3446 * Next the RX queues...
3448 rsize = roundup2(adapter->num_rx_desc *
3449 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3450 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3451 rxr = &adapter->rx_rings[i];
3452 rxr->adapter = adapter;
3455 /* Initialize the RX lock */
3456 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3457 device_get_nameunit(dev), txr->me);
3458 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3460 if (igb_dma_malloc(adapter, rsize,
3461 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3463 "Unable to allocate RxDescriptor memory\n");
3467 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3468 bzero((void *)rxr->rx_base, rsize);
3470 /* Allocate receive buffers for the ring*/
3471 if (igb_allocate_receive_buffers(rxr)) {
3473 "Critical Failure setting up receive buffers\n");
3480 ** Finally set up the queue holding structs
3482 for (int i = 0; i < adapter->num_queues; i++) {
3483 que = &adapter->queues[i];
3484 que->adapter = adapter;
3485 que->txr = &adapter->tx_rings[i];
3486 que->rxr = &adapter->rx_rings[i];
3492 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3493 igb_dma_free(adapter, &rxr->rxdma);
3495 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3496 igb_dma_free(adapter, &txr->txdma);
3497 free(adapter->rx_rings, M_DEVBUF);
3499 #ifndef IGB_LEGACY_TX
3500 buf_ring_free(txr->br, M_DEVBUF);
3502 free(adapter->tx_rings, M_DEVBUF);
3504 free(adapter->queues, M_DEVBUF);
3509 /*********************************************************************
3511 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3512 * the information needed to transmit a packet on the wire. This is
3513 * called only once at attach, setup is done every reset.
3515 **********************************************************************/
3517 igb_allocate_transmit_buffers(struct tx_ring *txr)
3519 struct adapter *adapter = txr->adapter;
3520 device_t dev = adapter->dev;
3521 struct igb_tx_buf *txbuf;
3525 * Setup DMA descriptor areas.
3527 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3528 1, 0, /* alignment, bounds */
3529 BUS_SPACE_MAXADDR, /* lowaddr */
3530 BUS_SPACE_MAXADDR, /* highaddr */
3531 NULL, NULL, /* filter, filterarg */
3532 IGB_TSO_SIZE, /* maxsize */
3533 IGB_MAX_SCATTER, /* nsegments */
3534 PAGE_SIZE, /* maxsegsize */
3536 NULL, /* lockfunc */
3537 NULL, /* lockfuncarg */
3539 device_printf(dev,"Unable to allocate TX DMA tag\n");
3543 if (!(txr->tx_buffers =
3544 (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
3545 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3546 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3551 /* Create the descriptor buffer dma maps */
3552 txbuf = txr->tx_buffers;
3553 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3554 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3556 device_printf(dev, "Unable to create TX DMA map\n");
3563 /* We free all, it handles case where we are in the middle */
3564 igb_free_transmit_structures(adapter);
3568 /*********************************************************************
3570 * Initialize a transmit ring.
3572 **********************************************************************/
3574 igb_setup_transmit_ring(struct tx_ring *txr)
3576 struct adapter *adapter = txr->adapter;
3577 struct igb_tx_buf *txbuf;
3580 struct netmap_adapter *na = NA(adapter->ifp);
3581 struct netmap_slot *slot;
3582 #endif /* DEV_NETMAP */
3584 /* Clear the old descriptor contents */
3587 slot = netmap_reset(na, NR_TX, txr->me, 0);
3588 #endif /* DEV_NETMAP */
3589 bzero((void *)txr->tx_base,
3590 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3592 txr->next_avail_desc = 0;
3593 txr->next_to_clean = 0;
3595 /* Free any existing tx buffers. */
3596 txbuf = txr->tx_buffers;
3597 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3598 if (txbuf->m_head != NULL) {
3599 bus_dmamap_sync(txr->txtag, txbuf->map,
3600 BUS_DMASYNC_POSTWRITE);
3601 bus_dmamap_unload(txr->txtag, txbuf->map);
3602 m_freem(txbuf->m_head);
3603 txbuf->m_head = NULL;
3607 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3608 /* no need to set the address */
3609 netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
3611 #endif /* DEV_NETMAP */
3612 /* clear the watch index */
3616 /* Set number of descriptors available */
3617 txr->tx_avail = adapter->num_tx_desc;
3619 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3620 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3624 /*********************************************************************
3626 * Initialize all transmit rings.
3628 **********************************************************************/
3630 igb_setup_transmit_structures(struct adapter *adapter)
3632 struct tx_ring *txr = adapter->tx_rings;
3634 for (int i = 0; i < adapter->num_queues; i++, txr++)
3635 igb_setup_transmit_ring(txr);
3640 /*********************************************************************
3642 * Enable transmit unit.
3644 **********************************************************************/
3646 igb_initialize_transmit_units(struct adapter *adapter)
3648 struct tx_ring *txr = adapter->tx_rings;
3649 struct e1000_hw *hw = &adapter->hw;
3652 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3655 /* Setup the Tx Descriptor Rings */
3656 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3657 u64 bus_addr = txr->txdma.dma_paddr;
3659 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3660 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3661 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3662 (uint32_t)(bus_addr >> 32));
3663 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3664 (uint32_t)bus_addr);
3666 /* Setup the HW Tx Head and Tail descriptor pointers */
3667 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3668 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3670 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3671 E1000_READ_REG(hw, E1000_TDBAL(i)),
3672 E1000_READ_REG(hw, E1000_TDLEN(i)));
3674 txr->queue_status = IGB_QUEUE_IDLE;
3676 txdctl |= IGB_TX_PTHRESH;
3677 txdctl |= IGB_TX_HTHRESH << 8;
3678 txdctl |= IGB_TX_WTHRESH << 16;
3679 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3680 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3683 if (adapter->vf_ifp)
3686 e1000_config_collision_dist(hw);
3688 /* Program the Transmit Control Register */
3689 tctl = E1000_READ_REG(hw, E1000_TCTL);
3690 tctl &= ~E1000_TCTL_CT;
3691 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3692 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3694 /* This write will effectively turn on the transmit unit. */
3695 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3698 /*********************************************************************
3700 * Free all transmit rings.
3702 **********************************************************************/
3704 igb_free_transmit_structures(struct adapter *adapter)
3706 struct tx_ring *txr = adapter->tx_rings;
3708 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3710 igb_free_transmit_buffers(txr);
3711 igb_dma_free(adapter, &txr->txdma);
3713 IGB_TX_LOCK_DESTROY(txr);
3715 free(adapter->tx_rings, M_DEVBUF);
3718 /*********************************************************************
3720 * Free transmit ring related data structures.
3722 **********************************************************************/
3724 igb_free_transmit_buffers(struct tx_ring *txr)
3726 struct adapter *adapter = txr->adapter;
3727 struct igb_tx_buf *tx_buffer;
3730 INIT_DEBUGOUT("free_transmit_ring: begin");
3732 if (txr->tx_buffers == NULL)
3735 tx_buffer = txr->tx_buffers;
3736 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3737 if (tx_buffer->m_head != NULL) {
3738 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3739 BUS_DMASYNC_POSTWRITE);
3740 bus_dmamap_unload(txr->txtag,
3742 m_freem(tx_buffer->m_head);
3743 tx_buffer->m_head = NULL;
3744 if (tx_buffer->map != NULL) {
3745 bus_dmamap_destroy(txr->txtag,
3747 tx_buffer->map = NULL;
3749 } else if (tx_buffer->map != NULL) {
3750 bus_dmamap_unload(txr->txtag,
3752 bus_dmamap_destroy(txr->txtag,
3754 tx_buffer->map = NULL;
3757 #ifndef IGB_LEGACY_TX
3758 if (txr->br != NULL)
3759 buf_ring_free(txr->br, M_DEVBUF);
3761 if (txr->tx_buffers != NULL) {
3762 free(txr->tx_buffers, M_DEVBUF);
3763 txr->tx_buffers = NULL;
3765 if (txr->txtag != NULL) {
3766 bus_dma_tag_destroy(txr->txtag);
3772 /**********************************************************************
3774 * Setup work for hardware segmentation offload (TSO) on
3775 * adapters using advanced tx descriptors
3777 **********************************************************************/
3779 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3780 u32 *cmd_type_len, u32 *olinfo_status)
3782 struct adapter *adapter = txr->adapter;
3783 struct e1000_adv_tx_context_desc *TXD;
3784 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3785 u32 mss_l4len_idx = 0, paylen;
3786 u16 vtag = 0, eh_type;
3787 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3788 struct ether_vlan_header *eh;
3790 struct ip6_hdr *ip6;
3799 * Determine where frame payload starts.
3800 * Jump over vlan headers if already present
3802 eh = mtod(mp, struct ether_vlan_header *);
3803 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3804 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3805 eh_type = eh->evl_proto;
3807 ehdrlen = ETHER_HDR_LEN;
3808 eh_type = eh->evl_encap_proto;
3811 switch (ntohs(eh_type)) {
3813 case ETHERTYPE_IPV6:
3814 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3815 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3816 if (ip6->ip6_nxt != IPPROTO_TCP)
3818 ip_hlen = sizeof(struct ip6_hdr);
3819 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3820 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3821 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3822 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3827 ip = (struct ip *)(mp->m_data + ehdrlen);
3828 if (ip->ip_p != IPPROTO_TCP)
3831 ip_hlen = ip->ip_hl << 2;
3832 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3833 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3834 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3835 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3836 /* Tell transmit desc to also do IPv4 checksum. */
3837 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3841 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3842 __func__, ntohs(eh_type));
3846 ctxd = txr->next_avail_desc;
3847 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3849 tcp_hlen = th->th_off << 2;
3851 /* This is used in the transmit desc in encap */
3852 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3854 /* VLAN MACLEN IPLEN */
3855 if (mp->m_flags & M_VLANTAG) {
3856 vtag = htole16(mp->m_pkthdr.ether_vtag);
3857 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3860 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3861 vlan_macip_lens |= ip_hlen;
3862 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3864 /* ADV DTYPE TUCMD */
3865 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3866 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3867 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3870 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3871 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3872 /* 82575 needs the queue index added */
3873 if (adapter->hw.mac.type == e1000_82575)
3874 mss_l4len_idx |= txr->me << 4;
3875 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3877 TXD->seqnum_seed = htole32(0);
3879 if (++ctxd == txr->num_desc)
3883 txr->next_avail_desc = ctxd;
3884 *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3885 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3886 *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
3891 /*********************************************************************
3893 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3895 **********************************************************************/
3898 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3899 u32 *cmd_type_len, u32 *olinfo_status)
3901 struct e1000_adv_tx_context_desc *TXD;
3902 struct adapter *adapter = txr->adapter;
3903 struct ether_vlan_header *eh;
3905 struct ip6_hdr *ip6;
3906 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
3907 int ehdrlen, ip_hlen = 0;
3911 int ctxd = txr->next_avail_desc;
3914 /* First check if TSO is to be used */
3915 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3916 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3918 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3921 /* Indicate the whole packet as payload when not doing TSO */
3922 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
3924 /* Now ready a context descriptor */
3925 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3928 ** In advanced descriptors the vlan tag must
3929 ** be placed into the context descriptor. Hence
3930 ** we need to make one even if not doing offloads.
3932 if (mp->m_flags & M_VLANTAG) {
3933 vtag = htole16(mp->m_pkthdr.ether_vtag);
3934 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3935 } else if (offload == FALSE) /* ... no offload to do */
3939 * Determine where frame payload starts.
3940 * Jump over vlan headers if already present,
3941 * helpful for QinQ too.
3943 eh = mtod(mp, struct ether_vlan_header *);
3944 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3945 etype = ntohs(eh->evl_proto);
3946 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3948 etype = ntohs(eh->evl_encap_proto);
3949 ehdrlen = ETHER_HDR_LEN;
3952 /* Set the ether header length */
3953 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3957 ip = (struct ip *)(mp->m_data + ehdrlen);
3958 ip_hlen = ip->ip_hl << 2;
3960 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3962 case ETHERTYPE_IPV6:
3963 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3964 ip_hlen = sizeof(struct ip6_hdr);
3965 /* XXX-BZ this will go badly in case of ext hdrs. */
3966 ipproto = ip6->ip6_nxt;
3967 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3974 vlan_macip_lens |= ip_hlen;
3975 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3979 #if __FreeBSD_version >= 1000000
3980 if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))
3982 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3984 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3987 #if __FreeBSD_version >= 1000000
3988 if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP))
3990 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3992 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3995 #if __FreeBSD_version >= 800000
3997 #if __FreeBSD_version >= 1000000
3998 if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP))
4000 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
4002 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4010 if (offload) /* For the TX descriptor setup */
4011 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4013 /* 82575 needs the queue index added */
4014 if (adapter->hw.mac.type == e1000_82575)
4015 mss_l4len_idx = txr->me << 4;
4017 /* Now copy bits into descriptor */
4018 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
4019 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
4020 TXD->seqnum_seed = htole32(0);
4021 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
4023 /* We've consumed the first desc, adjust counters */
4024 if (++ctxd == txr->num_desc)
4026 txr->next_avail_desc = ctxd;
4032 /**********************************************************************
4034 * Examine each tx_buffer in the used queue. If the hardware is done
4035 * processing the packet then free associated resources. The
4036 * tx_buffer is put back on the free queue.
4038 * TRUE return means there's work in the ring to clean, FALSE its empty.
4039 **********************************************************************/
4041 igb_txeof(struct tx_ring *txr)
4043 struct adapter *adapter = txr->adapter;
4045 struct ifnet *ifp = adapter->ifp;
4046 #endif /* DEV_NETMAP */
4047 u32 work, processed = 0;
4048 int limit = adapter->tx_process_limit;
4049 struct igb_tx_buf *buf;
4050 union e1000_adv_tx_desc *txd;
4052 mtx_assert(&txr->tx_mtx, MA_OWNED);
4055 if (netmap_tx_irq(ifp, txr->me))
4057 #endif /* DEV_NETMAP */
4059 if (txr->tx_avail == txr->num_desc) {
4060 txr->queue_status = IGB_QUEUE_IDLE;
4064 /* Get work starting point */
4065 work = txr->next_to_clean;
4066 buf = &txr->tx_buffers[work];
4067 txd = &txr->tx_base[work];
4068 work -= txr->num_desc; /* The distance to ring end */
4069 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4070 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4072 union e1000_adv_tx_desc *eop = buf->eop;
4073 if (eop == NULL) /* No work */
4076 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
4077 break; /* I/O not complete */
4081 buf->m_head->m_pkthdr.len;
4082 bus_dmamap_sync(txr->txtag,
4084 BUS_DMASYNC_POSTWRITE);
4085 bus_dmamap_unload(txr->txtag,
4087 m_freem(buf->m_head);
4093 /* We clean the range if multi segment */
4094 while (txd != eop) {
4098 /* wrap the ring? */
4099 if (__predict_false(!work)) {
4100 work -= txr->num_desc;
4101 buf = txr->tx_buffers;
4106 buf->m_head->m_pkthdr.len;
4107 bus_dmamap_sync(txr->txtag,
4109 BUS_DMASYNC_POSTWRITE);
4110 bus_dmamap_unload(txr->txtag,
4112 m_freem(buf->m_head);
4121 txr->watchdog_time = ticks;
4123 /* Try the next packet */
4127 /* reset with a wrap */
4128 if (__predict_false(!work)) {
4129 work -= txr->num_desc;
4130 buf = txr->tx_buffers;
4134 } while (__predict_true(--limit));
4136 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4137 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4139 work += txr->num_desc;
4140 txr->next_to_clean = work;
4143 ** Watchdog calculation, we know there's
4144 ** work outstanding or the first return
4145 ** would have been taken, so none processed
4146 ** for too long indicates a hang.
4148 if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
4149 txr->queue_status |= IGB_QUEUE_HUNG;
4151 if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
4152 txr->queue_status &= ~IGB_QUEUE_DEPLETED;
4154 if (txr->tx_avail == txr->num_desc) {
4155 txr->queue_status = IGB_QUEUE_IDLE;
4162 /*********************************************************************
4164 * Refresh mbuf buffers for RX descriptor rings
4165 * - now keeps its own state so discards due to resource
4166 * exhaustion are unnecessary, if an mbuf cannot be obtained
4167 * it just returns, keeping its placeholder, thus it can simply
4168 * be recalled to try again.
4170 **********************************************************************/
4172 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
4174 struct adapter *adapter = rxr->adapter;
4175 bus_dma_segment_t hseg[1];
4176 bus_dma_segment_t pseg[1];
4177 struct igb_rx_buf *rxbuf;
4178 struct mbuf *mh, *mp;
4179 int i, j, nsegs, error;
4180 bool refreshed = FALSE;
4182 i = j = rxr->next_to_refresh;
4184 ** Get one descriptor beyond
4185 ** our work mark to control
4188 if (++j == adapter->num_rx_desc)
4191 while (j != limit) {
4192 rxbuf = &rxr->rx_buffers[i];
4193 /* No hdr mbuf used with header split off */
4194 if (rxr->hdr_split == FALSE)
4196 if (rxbuf->m_head == NULL) {
4197 mh = m_gethdr(M_NOWAIT, MT_DATA);
4203 mh->m_pkthdr.len = mh->m_len = MHLEN;
4205 mh->m_flags |= M_PKTHDR;
4206 /* Get the memory mapping */
4207 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4208 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
4210 printf("Refresh mbufs: hdr dmamap load"
4211 " failure - %d\n", error);
4213 rxbuf->m_head = NULL;
4217 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4218 BUS_DMASYNC_PREREAD);
4219 rxr->rx_base[i].read.hdr_addr =
4220 htole64(hseg[0].ds_addr);
4222 if (rxbuf->m_pack == NULL) {
4223 mp = m_getjcl(M_NOWAIT, MT_DATA,
4224 M_PKTHDR, adapter->rx_mbuf_sz);
4230 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4231 /* Get the memory mapping */
4232 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4233 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
4235 printf("Refresh mbufs: payload dmamap load"
4236 " failure - %d\n", error);
4238 rxbuf->m_pack = NULL;
4242 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4243 BUS_DMASYNC_PREREAD);
4244 rxr->rx_base[i].read.pkt_addr =
4245 htole64(pseg[0].ds_addr);
4246 refreshed = TRUE; /* I feel wefreshed :) */
4248 i = j; /* our next is precalculated */
4249 rxr->next_to_refresh = i;
4250 if (++j == adapter->num_rx_desc)
4254 if (refreshed) /* update tail */
4255 E1000_WRITE_REG(&adapter->hw,
4256 E1000_RDT(rxr->me), rxr->next_to_refresh);
4261 /*********************************************************************
4263 * Allocate memory for rx_buffer structures. Since we use one
4264 * rx_buffer per received packet, the maximum number of rx_buffer's
4265 * that we'll need is equal to the number of receive descriptors
4266 * that we've allocated.
4268 **********************************************************************/
4270 igb_allocate_receive_buffers(struct rx_ring *rxr)
4272 struct adapter *adapter = rxr->adapter;
4273 device_t dev = adapter->dev;
4274 struct igb_rx_buf *rxbuf;
4275 int i, bsize, error;
4277 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
4278 if (!(rxr->rx_buffers =
4279 (struct igb_rx_buf *) malloc(bsize,
4280 M_DEVBUF, M_NOWAIT | M_ZERO))) {
4281 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4286 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4287 1, 0, /* alignment, bounds */
4288 BUS_SPACE_MAXADDR, /* lowaddr */
4289 BUS_SPACE_MAXADDR, /* highaddr */
4290 NULL, NULL, /* filter, filterarg */
4291 MSIZE, /* maxsize */
4293 MSIZE, /* maxsegsize */
4295 NULL, /* lockfunc */
4296 NULL, /* lockfuncarg */
4298 device_printf(dev, "Unable to create RX DMA tag\n");
4302 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4303 1, 0, /* alignment, bounds */
4304 BUS_SPACE_MAXADDR, /* lowaddr */
4305 BUS_SPACE_MAXADDR, /* highaddr */
4306 NULL, NULL, /* filter, filterarg */
4307 MJUM9BYTES, /* maxsize */
4309 MJUM9BYTES, /* maxsegsize */
4311 NULL, /* lockfunc */
4312 NULL, /* lockfuncarg */
4314 device_printf(dev, "Unable to create RX payload DMA tag\n");
4318 for (i = 0; i < adapter->num_rx_desc; i++) {
4319 rxbuf = &rxr->rx_buffers[i];
4320 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
4323 "Unable to create RX head DMA maps\n");
4326 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
4329 "Unable to create RX packet DMA maps\n");
4337 /* Frees all, but can handle partial completion */
4338 igb_free_receive_structures(adapter);
4344 igb_free_receive_ring(struct rx_ring *rxr)
4346 struct adapter *adapter = rxr->adapter;
4347 struct igb_rx_buf *rxbuf;
4350 for (int i = 0; i < adapter->num_rx_desc; i++) {
4351 rxbuf = &rxr->rx_buffers[i];
4352 if (rxbuf->m_head != NULL) {
4353 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4354 BUS_DMASYNC_POSTREAD);
4355 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4356 rxbuf->m_head->m_flags |= M_PKTHDR;
4357 m_freem(rxbuf->m_head);
4359 if (rxbuf->m_pack != NULL) {
4360 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4361 BUS_DMASYNC_POSTREAD);
4362 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4363 rxbuf->m_pack->m_flags |= M_PKTHDR;
4364 m_freem(rxbuf->m_pack);
4366 rxbuf->m_head = NULL;
4367 rxbuf->m_pack = NULL;
4372 /*********************************************************************
4374 * Initialize a receive ring and its buffers.
4376 **********************************************************************/
4378 igb_setup_receive_ring(struct rx_ring *rxr)
4380 struct adapter *adapter;
4383 struct igb_rx_buf *rxbuf;
4384 bus_dma_segment_t pseg[1], hseg[1];
4385 struct lro_ctrl *lro = &rxr->lro;
4386 int rsize, nsegs, error = 0;
4388 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4389 struct netmap_slot *slot;
4390 #endif /* DEV_NETMAP */
4392 adapter = rxr->adapter;
4396 /* Clear the ring contents */
4399 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4400 #endif /* DEV_NETMAP */
4401 rsize = roundup2(adapter->num_rx_desc *
4402 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
4403 bzero((void *)rxr->rx_base, rsize);
4406 ** Free current RX buffer structures and their mbufs
4408 igb_free_receive_ring(rxr);
4410 /* Configure for header split? */
4411 if (igb_header_split)
4412 rxr->hdr_split = TRUE;
4414 /* Now replenish the ring mbufs */
4415 for (int j = 0; j < adapter->num_rx_desc; ++j) {
4416 struct mbuf *mh, *mp;
4418 rxbuf = &rxr->rx_buffers[j];
4421 /* slot sj is mapped to the j-th NIC-ring entry */
4422 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4426 addr = PNMB(na, slot + sj, &paddr);
4427 netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
4428 /* Update descriptor */
4429 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4432 #endif /* DEV_NETMAP */
4433 if (rxr->hdr_split == FALSE)
4436 /* First the header */
4437 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4438 if (rxbuf->m_head == NULL) {
4442 m_adj(rxbuf->m_head, ETHER_ALIGN);
4444 mh->m_len = mh->m_pkthdr.len = MHLEN;
4445 mh->m_flags |= M_PKTHDR;
4446 /* Get the memory mapping */
4447 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4448 rxbuf->hmap, rxbuf->m_head, hseg,
4449 &nsegs, BUS_DMA_NOWAIT);
4450 if (error != 0) /* Nothing elegant to do here */
4452 bus_dmamap_sync(rxr->htag,
4453 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4454 /* Update descriptor */
4455 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4458 /* Now the payload cluster */
4459 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4460 M_PKTHDR, adapter->rx_mbuf_sz);
4461 if (rxbuf->m_pack == NULL) {
4466 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4467 /* Get the memory mapping */
4468 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4469 rxbuf->pmap, mp, pseg,
4470 &nsegs, BUS_DMA_NOWAIT);
4473 bus_dmamap_sync(rxr->ptag,
4474 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4475 /* Update descriptor */
4476 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4479 /* Setup our descriptor indices */
4480 rxr->next_to_check = 0;
4481 rxr->next_to_refresh = adapter->num_rx_desc - 1;
4482 rxr->lro_enabled = FALSE;
4483 rxr->rx_split_packets = 0;
4489 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4490 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4493 ** Now set up the LRO interface, we
4494 ** also only do head split when LRO
4495 ** is enabled, since so often they
4496 ** are undesirable in similar setups.
4498 if (ifp->if_capenable & IFCAP_LRO) {
4499 error = tcp_lro_init(lro);
4501 device_printf(dev, "LRO Initialization failed!\n");
4504 INIT_DEBUGOUT("RX LRO Initialized\n");
4505 rxr->lro_enabled = TRUE;
4506 lro->ifp = adapter->ifp;
4513 igb_free_receive_ring(rxr);
4519 /*********************************************************************
4521 * Initialize all receive rings.
4523 **********************************************************************/
4525 igb_setup_receive_structures(struct adapter *adapter)
4527 struct rx_ring *rxr = adapter->rx_rings;
4530 for (i = 0; i < adapter->num_queues; i++, rxr++)
4531 if (igb_setup_receive_ring(rxr))
4537 * Free RX buffers allocated so far, we will only handle
4538 * the rings that completed, the failing case will have
4539 * cleaned up for itself. 'i' is the endpoint.
4541 for (int j = 0; j < i; ++j) {
4542 rxr = &adapter->rx_rings[j];
4544 igb_free_receive_ring(rxr);
4552 * Initialise the RSS mapping for NICs that support multiple transmit/
4556 igb_initialise_rss_mapping(struct adapter *adapter)
4558 struct e1000_hw *hw = &adapter->hw;
4562 u32 rss_key[10], mrqc, shift = 0;
4565 if (adapter->hw.mac.type == e1000_82575)
4569 * The redirection table controls which destination
4570 * queue each bucket redirects traffic to.
4571 * Each DWORD represents four queues, with the LSB
4572 * being the first queue in the DWORD.
4574 * This just allocates buckets to queues using round-robin
4577 * NOTE: It Just Happens to line up with the default
4578 * RSS allocation method.
4581 /* Warning FM follows */
4583 for (i = 0; i < 128; i++) {
4585 queue_id = rss_get_indirection_to_bucket(i);
4587 * If we have more queues than buckets, we'll
4588 * end up mapping buckets to a subset of the
4591 * If we have more buckets than queues, we'll
4592 * end up instead assigning multiple buckets
4595 * Both are suboptimal, but we need to handle
4596 * the case so we don't go out of bounds
4597 * indexing arrays and such.
4599 queue_id = queue_id % adapter->num_queues;
4601 queue_id = (i % adapter->num_queues);
4603 /* Adjust if required */
4604 queue_id = queue_id << shift;
4607 * The low 8 bits are for hash value (n+0);
4608 * The next 8 bits are for hash value (n+1), etc.
4611 reta = reta | ( ((uint32_t) queue_id) << 24);
4613 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
4618 /* Now fill in hash table */
4621 * MRQC: Multiple Receive Queues Command
4622 * Set queuing to RSS control, number depends on the device.
4624 mrqc = E1000_MRQC_ENABLE_RSS_8Q;
4627 /* XXX ew typecasting */
4628 rss_getkey((uint8_t *) &rss_key);
4630 arc4rand(&rss_key, sizeof(rss_key), 0);
4632 for (i = 0; i < 10; i++)
4633 E1000_WRITE_REG_ARRAY(hw,
4634 E1000_RSSRK(0), i, rss_key[i]);
4637 * Configure the RSS fields to hash upon.
4639 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
4640 E1000_MRQC_RSS_FIELD_IPV4_TCP);
4641 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
4642 E1000_MRQC_RSS_FIELD_IPV6_TCP);
4643 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
4644 E1000_MRQC_RSS_FIELD_IPV6_UDP);
4645 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
4646 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
4648 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
4651 /*********************************************************************
4653 * Enable receive unit.
4655 **********************************************************************/
4657 igb_initialize_receive_units(struct adapter *adapter)
4659 struct rx_ring *rxr = adapter->rx_rings;
4660 struct ifnet *ifp = adapter->ifp;
4661 struct e1000_hw *hw = &adapter->hw;
4662 u32 rctl, rxcsum, psize, srrctl = 0;
4664 INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
4667 * Make sure receives are disabled while setting
4668 * up the descriptor ring
4670 rctl = E1000_READ_REG(hw, E1000_RCTL);
4671 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4674 ** Set up for header split
4676 if (igb_header_split) {
4677 /* Use a standard mbuf for the header */
4678 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4679 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4681 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4684 ** Set up for jumbo frames
4686 if (ifp->if_mtu > ETHERMTU) {
4687 rctl |= E1000_RCTL_LPE;
4688 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
4689 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4690 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4691 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
4692 srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4693 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4695 /* Set maximum packet len */
4696 psize = adapter->max_frame_size;
4697 /* are we on a vlan? */
4698 if (adapter->ifp->if_vlantrunk != NULL)
4699 psize += VLAN_TAG_SIZE;
4700 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
4702 rctl &= ~E1000_RCTL_LPE;
4703 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4704 rctl |= E1000_RCTL_SZ_2048;
4708 * If TX flow control is disabled and there's >1 queue defined,
4711 * This drops frames rather than hanging the RX MAC for all queues.
4713 if ((adapter->num_queues > 1) &&
4714 (adapter->fc == e1000_fc_none ||
4715 adapter->fc == e1000_fc_rx_pause)) {
4716 srrctl |= E1000_SRRCTL_DROP_EN;
4719 /* Setup the Base and Length of the Rx Descriptor Rings */
4720 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4721 u64 bus_addr = rxr->rxdma.dma_paddr;
4724 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4725 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4726 E1000_WRITE_REG(hw, E1000_RDBAH(i),
4727 (uint32_t)(bus_addr >> 32));
4728 E1000_WRITE_REG(hw, E1000_RDBAL(i),
4729 (uint32_t)bus_addr);
4730 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
4731 /* Enable this Queue */
4732 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
4733 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4734 rxdctl &= 0xFFF00000;
4735 rxdctl |= IGB_RX_PTHRESH;
4736 rxdctl |= IGB_RX_HTHRESH << 8;
4737 rxdctl |= IGB_RX_WTHRESH << 16;
4738 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
4742 ** Setup for RX MultiQueue
4744 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4745 if (adapter->num_queues >1) {
4748 igb_initialise_rss_mapping(adapter);
4751 ** NOTE: Receive Full-Packet Checksum Offload
4752 ** is mutually exclusive with Multiqueue. However
4753 ** this is not the same as TCP/IP checksums which
4756 rxcsum |= E1000_RXCSUM_PCSD;
4757 #if __FreeBSD_version >= 800000
4758 /* For SCTP Offload */
4759 if ((hw->mac.type != e1000_82575) &&
4760 (ifp->if_capenable & IFCAP_RXCSUM))
4761 rxcsum |= E1000_RXCSUM_CRCOFL;
4765 if (ifp->if_capenable & IFCAP_RXCSUM) {
4766 rxcsum |= E1000_RXCSUM_IPPCSE;
4767 #if __FreeBSD_version >= 800000
4768 if (adapter->hw.mac.type != e1000_82575)
4769 rxcsum |= E1000_RXCSUM_CRCOFL;
4772 rxcsum &= ~E1000_RXCSUM_TUOFL;
4774 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4776 /* Setup the Receive Control Register */
4777 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4778 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4779 E1000_RCTL_RDMTS_HALF |
4780 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4781 /* Strip CRC bytes. */
4782 rctl |= E1000_RCTL_SECRC;
4783 /* Make sure VLAN Filters are off */
4784 rctl &= ~E1000_RCTL_VFE;
4785 /* Don't store bad packets */
4786 rctl &= ~E1000_RCTL_SBP;
4788 /* Enable Receives */
4789 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4792 * Setup the HW Rx Head and Tail Descriptor Pointers
4793 * - needs to be after enable
4795 for (int i = 0; i < adapter->num_queues; i++) {
4796 rxr = &adapter->rx_rings[i];
4797 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
4800 * an init() while a netmap client is active must
4801 * preserve the rx buffers passed to userspace.
4802 * In this driver it means we adjust RDT to
4803 * something different from next_to_refresh
4804 * (which is not used in netmap mode).
4806 if (ifp->if_capenable & IFCAP_NETMAP) {
4807 struct netmap_adapter *na = NA(adapter->ifp);
4808 struct netmap_kring *kring = &na->rx_rings[i];
4809 int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
4811 if (t >= adapter->num_rx_desc)
4812 t -= adapter->num_rx_desc;
4814 t += adapter->num_rx_desc;
4815 E1000_WRITE_REG(hw, E1000_RDT(i), t);
4817 #endif /* DEV_NETMAP */
4818 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
4823 /*********************************************************************
4825 * Free receive rings.
4827 **********************************************************************/
4829 igb_free_receive_structures(struct adapter *adapter)
4831 struct rx_ring *rxr = adapter->rx_rings;
4833 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4834 struct lro_ctrl *lro = &rxr->lro;
4835 igb_free_receive_buffers(rxr);
4837 igb_dma_free(adapter, &rxr->rxdma);
4840 free(adapter->rx_rings, M_DEVBUF);
4843 /*********************************************************************
4845 * Free receive ring data structures.
4847 **********************************************************************/
4849 igb_free_receive_buffers(struct rx_ring *rxr)
4851 struct adapter *adapter = rxr->adapter;
4852 struct igb_rx_buf *rxbuf;
4855 INIT_DEBUGOUT("free_receive_structures: begin");
4857 /* Cleanup any existing buffers */
4858 if (rxr->rx_buffers != NULL) {
4859 for (i = 0; i < adapter->num_rx_desc; i++) {
4860 rxbuf = &rxr->rx_buffers[i];
4861 if (rxbuf->m_head != NULL) {
4862 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4863 BUS_DMASYNC_POSTREAD);
4864 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4865 rxbuf->m_head->m_flags |= M_PKTHDR;
4866 m_freem(rxbuf->m_head);
4868 if (rxbuf->m_pack != NULL) {
4869 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4870 BUS_DMASYNC_POSTREAD);
4871 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4872 rxbuf->m_pack->m_flags |= M_PKTHDR;
4873 m_freem(rxbuf->m_pack);
4875 rxbuf->m_head = NULL;
4876 rxbuf->m_pack = NULL;
4877 if (rxbuf->hmap != NULL) {
4878 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4881 if (rxbuf->pmap != NULL) {
4882 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4886 if (rxr->rx_buffers != NULL) {
4887 free(rxr->rx_buffers, M_DEVBUF);
4888 rxr->rx_buffers = NULL;
4892 if (rxr->htag != NULL) {
4893 bus_dma_tag_destroy(rxr->htag);
4896 if (rxr->ptag != NULL) {
4897 bus_dma_tag_destroy(rxr->ptag);
4902 static __inline void
4903 igb_rx_discard(struct rx_ring *rxr, int i)
4905 struct igb_rx_buf *rbuf;
4907 rbuf = &rxr->rx_buffers[i];
4909 /* Partially received? Free the chain */
4910 if (rxr->fmp != NULL) {
4911 rxr->fmp->m_flags |= M_PKTHDR;
4918 ** With advanced descriptors the writeback
4919 ** clobbers the buffer addrs, so its easier
4920 ** to just free the existing mbufs and take
4921 ** the normal refresh path to get new buffers
4925 m_free(rbuf->m_head);
4926 rbuf->m_head = NULL;
4927 bus_dmamap_unload(rxr->htag, rbuf->hmap);
4931 m_free(rbuf->m_pack);
4932 rbuf->m_pack = NULL;
4933 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
4939 static __inline void
4940 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4944 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4945 * should be computed by hardware. Also it should not have VLAN tag in
4948 if (rxr->lro_enabled &&
4949 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4950 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4951 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4952 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4953 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4954 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4956 * Send to the stack if:
4957 ** - LRO not enabled, or
4958 ** - no LRO resources, or
4959 ** - lro enqueue fails
4961 if (rxr->lro.lro_cnt != 0)
4962 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4966 (*ifp->if_input)(ifp, m);
4970 /*********************************************************************
4972 * This routine executes in interrupt context. It replenishes
4973 * the mbufs in the descriptor and sends data which has been
4974 * dma'ed into host memory to upper layer.
4976 * We loop at most count times if count is > 0, or until done if
4979 * Return TRUE if more to clean, FALSE otherwise
4980 *********************************************************************/
4982 igb_rxeof(struct igb_queue *que, int count, int *done)
4984 struct adapter *adapter = que->adapter;
4985 struct rx_ring *rxr = que->rxr;
4986 struct ifnet *ifp = adapter->ifp;
4987 struct lro_ctrl *lro = &rxr->lro;
4988 int i, processed = 0, rxdone = 0;
4989 u32 ptype, staterr = 0;
4990 union e1000_adv_rx_desc *cur;
4993 /* Sync the ring. */
4994 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4995 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4998 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
5002 #endif /* DEV_NETMAP */
5004 /* Main clean loop */
5005 for (i = rxr->next_to_check; count != 0;) {
5006 struct mbuf *sendmp, *mh, *mp;
5007 struct igb_rx_buf *rxbuf;
5008 u16 hlen, plen, hdr, vtag, pkt_info;
5011 cur = &rxr->rx_base[i];
5012 staterr = le32toh(cur->wb.upper.status_error);
5013 if ((staterr & E1000_RXD_STAT_DD) == 0)
5015 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
5018 sendmp = mh = mp = NULL;
5019 cur->wb.upper.status_error = 0;
5020 rxbuf = &rxr->rx_buffers[i];
5021 plen = le16toh(cur->wb.upper.length);
5022 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
5023 if (((adapter->hw.mac.type == e1000_i350) ||
5024 (adapter->hw.mac.type == e1000_i354)) &&
5025 (staterr & E1000_RXDEXT_STATERR_LB))
5026 vtag = be16toh(cur->wb.upper.vlan);
5028 vtag = le16toh(cur->wb.upper.vlan);
5029 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
5030 pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
5031 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
5034 * Free the frame (all segments) if we're at EOP and
5037 * The datasheet states that EOP + status is only valid for
5038 * the final segment in a multi-segment frame.
5040 if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
5041 adapter->dropped_pkts++;
5042 ++rxr->rx_discarded;
5043 igb_rx_discard(rxr, i);
5048 ** The way the hardware is configured to
5049 ** split, it will ONLY use the header buffer
5050 ** when header split is enabled, otherwise we
5051 ** get normal behavior, ie, both header and
5052 ** payload are DMA'd into the payload buffer.
5054 ** The fmp test is to catch the case where a
5055 ** packet spans multiple descriptors, in that
5056 ** case only the first header is valid.
5058 if (rxr->hdr_split && rxr->fmp == NULL) {
5059 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
5060 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
5061 E1000_RXDADV_HDRBUFLEN_SHIFT;
5062 if (hlen > IGB_HDR_BUF)
5064 mh = rxr->rx_buffers[i].m_head;
5066 /* clear buf pointer for refresh */
5067 rxbuf->m_head = NULL;
5069 ** Get the payload length, this
5070 ** could be zero if its a small
5074 mp = rxr->rx_buffers[i].m_pack;
5077 /* clear buf pointer */
5078 rxbuf->m_pack = NULL;
5079 rxr->rx_split_packets++;
5083 ** Either no header split, or a
5084 ** secondary piece of a fragmented
5087 mh = rxr->rx_buffers[i].m_pack;
5089 /* clear buf info for refresh */
5090 rxbuf->m_pack = NULL;
5092 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
5094 ++processed; /* So we know when to refresh */
5096 /* Initial frame - setup */
5097 if (rxr->fmp == NULL) {
5098 mh->m_pkthdr.len = mh->m_len;
5099 /* Save the head of the chain */
5103 /* Add payload if split */
5104 mh->m_pkthdr.len += mp->m_len;
5105 rxr->lmp = mh->m_next;
5108 /* Chain mbuf's together */
5109 rxr->lmp->m_next = mh;
5110 rxr->lmp = rxr->lmp->m_next;
5111 rxr->fmp->m_pkthdr.len += mh->m_len;
5115 rxr->fmp->m_pkthdr.rcvif = ifp;
5117 /* capture data for AIM */
5119 rxr->bytes += rxr->fmp->m_pkthdr.len;
5120 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
5122 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
5123 igb_rx_checksum(staterr, rxr->fmp, ptype);
5125 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
5126 (staterr & E1000_RXD_STAT_VP) != 0) {
5127 rxr->fmp->m_pkthdr.ether_vtag = vtag;
5128 rxr->fmp->m_flags |= M_VLANTAG;
5132 * In case of multiqueue, we have RXCSUM.PCSD bit set
5133 * and never cleared. This means we have RSS hash
5134 * available to be used.
5136 if (adapter->num_queues > 1) {
5137 rxr->fmp->m_pkthdr.flowid =
5138 le32toh(cur->wb.lower.hi_dword.rss);
5139 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
5140 case E1000_RXDADV_RSSTYPE_IPV4_TCP:
5141 M_HASHTYPE_SET(rxr->fmp,
5142 M_HASHTYPE_RSS_TCP_IPV4);
5144 case E1000_RXDADV_RSSTYPE_IPV4:
5145 M_HASHTYPE_SET(rxr->fmp,
5146 M_HASHTYPE_RSS_IPV4);
5148 case E1000_RXDADV_RSSTYPE_IPV6_TCP:
5149 M_HASHTYPE_SET(rxr->fmp,
5150 M_HASHTYPE_RSS_TCP_IPV6);
5152 case E1000_RXDADV_RSSTYPE_IPV6_EX:
5153 M_HASHTYPE_SET(rxr->fmp,
5154 M_HASHTYPE_RSS_IPV6_EX);
5156 case E1000_RXDADV_RSSTYPE_IPV6:
5157 M_HASHTYPE_SET(rxr->fmp,
5158 M_HASHTYPE_RSS_IPV6);
5160 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
5161 M_HASHTYPE_SET(rxr->fmp,
5162 M_HASHTYPE_RSS_TCP_IPV6_EX);
5165 /* XXX fallthrough */
5166 M_HASHTYPE_SET(rxr->fmp,
5167 M_HASHTYPE_OPAQUE_HASH);
5170 #ifndef IGB_LEGACY_TX
5171 rxr->fmp->m_pkthdr.flowid = que->msix;
5172 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
5176 /* Make sure to set M_PKTHDR. */
5177 sendmp->m_flags |= M_PKTHDR;
5183 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
5184 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5186 /* Advance our pointers to the next descriptor. */
5187 if (++i == adapter->num_rx_desc)
5190 ** Send to the stack or LRO
5192 if (sendmp != NULL) {
5193 rxr->next_to_check = i;
5194 igb_rx_input(rxr, ifp, sendmp, ptype);
5195 i = rxr->next_to_check;
5199 /* Every 8 descriptors we go to refresh mbufs */
5200 if (processed == 8) {
5201 igb_refresh_mbufs(rxr, i);
5206 /* Catch any remainders */
5207 if (igb_rx_unrefreshed(rxr))
5208 igb_refresh_mbufs(rxr, i);
5210 rxr->next_to_check = i;
5213 * Flush any outstanding LRO work
5215 tcp_lro_flush_all(lro);
5221 return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
5224 /*********************************************************************
5226 * Verify that the hardware indicated that the checksum is valid.
5227 * Inform the stack about the status of checksum so that stack
5228 * doesn't spend time verifying the checksum.
5230 *********************************************************************/
5232 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
5234 u16 status = (u16)staterr;
5235 u8 errors = (u8) (staterr >> 24);
5238 /* Ignore Checksum bit is set */
5239 if (status & E1000_RXD_STAT_IXSM) {
5240 mp->m_pkthdr.csum_flags = 0;
5244 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
5245 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
5249 if (status & E1000_RXD_STAT_IPCS) {
5251 if (!(errors & E1000_RXD_ERR_IPE)) {
5252 /* IP Checksum Good */
5253 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
5254 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
5256 mp->m_pkthdr.csum_flags = 0;
5259 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
5260 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5261 #if __FreeBSD_version >= 800000
5262 if (sctp) /* reassign */
5263 type = CSUM_SCTP_VALID;
5266 if (!(errors & E1000_RXD_ERR_TCPE)) {
5267 mp->m_pkthdr.csum_flags |= type;
5269 mp->m_pkthdr.csum_data = htons(0xffff);
5276 * This routine is run via an vlan
5280 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5282 struct adapter *adapter = ifp->if_softc;
5285 if (ifp->if_softc != arg) /* Not our event */
5288 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5291 IGB_CORE_LOCK(adapter);
5292 index = (vtag >> 5) & 0x7F;
5294 adapter->shadow_vfta[index] |= (1 << bit);
5295 ++adapter->num_vlans;
5296 /* Change hw filter setting */
5297 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5298 igb_setup_vlan_hw_support(adapter);
5299 IGB_CORE_UNLOCK(adapter);
5303 * This routine is run via an vlan
5307 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5309 struct adapter *adapter = ifp->if_softc;
5312 if (ifp->if_softc != arg)
5315 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5318 IGB_CORE_LOCK(adapter);
5319 index = (vtag >> 5) & 0x7F;
5321 adapter->shadow_vfta[index] &= ~(1 << bit);
5322 --adapter->num_vlans;
5323 /* Change hw filter setting */
5324 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5325 igb_setup_vlan_hw_support(adapter);
5326 IGB_CORE_UNLOCK(adapter);
5330 igb_setup_vlan_hw_support(struct adapter *adapter)
5332 struct e1000_hw *hw = &adapter->hw;
5333 struct ifnet *ifp = adapter->ifp;
5336 if (adapter->vf_ifp) {
5337 e1000_rlpml_set_vf(hw,
5338 adapter->max_frame_size + VLAN_TAG_SIZE);
5342 reg = E1000_READ_REG(hw, E1000_CTRL);
5343 reg |= E1000_CTRL_VME;
5344 E1000_WRITE_REG(hw, E1000_CTRL, reg);
5346 /* Enable the Filter Table */
5347 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
5348 reg = E1000_READ_REG(hw, E1000_RCTL);
5349 reg &= ~E1000_RCTL_CFIEN;
5350 reg |= E1000_RCTL_VFE;
5351 E1000_WRITE_REG(hw, E1000_RCTL, reg);
5354 /* Update the frame size */
5355 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
5356 adapter->max_frame_size + VLAN_TAG_SIZE);
5358 /* Don't bother with table if no vlans */
5359 if ((adapter->num_vlans == 0) ||
5360 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
5363 ** A soft reset zero's out the VFTA, so
5364 ** we need to repopulate it now.
5366 for (int i = 0; i < IGB_VFTA_SIZE; i++)
5367 if (adapter->shadow_vfta[i] != 0) {
5368 if (adapter->vf_ifp)
5369 e1000_vfta_set_vf(hw,
5370 adapter->shadow_vfta[i], TRUE);
5372 e1000_write_vfta(hw,
5373 i, adapter->shadow_vfta[i]);
5378 igb_enable_intr(struct adapter *adapter)
5380 /* With RSS set up what to auto clear */
5381 if (adapter->msix_mem) {
5382 u32 mask = (adapter->que_mask | adapter->link_mask);
5383 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
5384 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
5385 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
5386 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5389 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5392 E1000_WRITE_FLUSH(&adapter->hw);
5398 igb_disable_intr(struct adapter *adapter)
5400 if (adapter->msix_mem) {
5401 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
5402 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
5404 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
5405 E1000_WRITE_FLUSH(&adapter->hw);
5410 * Bit of a misnomer, what this really means is
5411 * to enable OS management of the system... aka
5412 * to disable special hardware management features
5415 igb_init_manageability(struct adapter *adapter)
5417 if (adapter->has_manage) {
5418 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
5419 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5421 /* disable hardware interception of ARP */
5422 manc &= ~(E1000_MANC_ARP_EN);
5424 /* enable receiving management packets to the host */
5425 manc |= E1000_MANC_EN_MNG2HOST;
5426 manc2h |= 1 << 5; /* Mng Port 623 */
5427 manc2h |= 1 << 6; /* Mng Port 664 */
5428 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
5429 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5434 * Give control back to hardware management
5435 * controller if there is one.
5438 igb_release_manageability(struct adapter *adapter)
5440 if (adapter->has_manage) {
5441 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5443 /* re-enable hardware interception of ARP */
5444 manc |= E1000_MANC_ARP_EN;
5445 manc &= ~E1000_MANC_EN_MNG2HOST;
5447 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5452 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
5453 * For ASF and Pass Through versions of f/w this means that
5454 * the driver is loaded.
5458 igb_get_hw_control(struct adapter *adapter)
5462 if (adapter->vf_ifp)
5465 /* Let firmware know the driver has taken over */
5466 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5467 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5468 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
5472 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
5473 * For ASF and Pass Through versions of f/w this means that the
5474 * driver is no longer loaded.
5478 igb_release_hw_control(struct adapter *adapter)
5482 if (adapter->vf_ifp)
5485 /* Let firmware taken over control of h/w */
5486 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5487 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5488 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
5492 igb_is_valid_ether_addr(uint8_t *addr)
5494 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
5496 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
5505 * Enable PCI Wake On Lan capability
5508 igb_enable_wakeup(device_t dev)
5513 /* First find the capabilities pointer*/
5514 cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
5515 /* Read the PM Capabilities */
5516 id = pci_read_config(dev, cap, 1);
5517 if (id != PCIY_PMG) /* Something wrong */
5519 /* OK, we have the power capabilities, so
5520 now get the status register */
5521 cap += PCIR_POWER_STATUS;
5522 status = pci_read_config(dev, cap, 2);
5523 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5524 pci_write_config(dev, cap, status, 2);
5529 igb_led_func(void *arg, int onoff)
5531 struct adapter *adapter = arg;
5533 IGB_CORE_LOCK(adapter);
5535 e1000_setup_led(&adapter->hw);
5536 e1000_led_on(&adapter->hw);
5538 e1000_led_off(&adapter->hw);
5539 e1000_cleanup_led(&adapter->hw);
5541 IGB_CORE_UNLOCK(adapter);
5545 igb_get_vf_counter(if_t ifp, ift_counter cnt)
5547 struct adapter *adapter;
5548 struct e1000_vf_stats *stats;
5549 #ifndef IGB_LEGACY_TX
5550 struct tx_ring *txr;
5554 adapter = if_getsoftc(ifp);
5555 stats = (struct e1000_vf_stats *)adapter->stats;
5558 case IFCOUNTER_IPACKETS:
5559 return (stats->gprc);
5560 case IFCOUNTER_OPACKETS:
5561 return (stats->gptc);
5562 case IFCOUNTER_IBYTES:
5563 return (stats->gorc);
5564 case IFCOUNTER_OBYTES:
5565 return (stats->gotc);
5566 case IFCOUNTER_IMCASTS:
5567 return (stats->mprc);
5568 case IFCOUNTER_IERRORS:
5569 return (adapter->dropped_pkts);
5570 case IFCOUNTER_OERRORS:
5571 return (adapter->watchdog_events);
5572 #ifndef IGB_LEGACY_TX
5573 case IFCOUNTER_OQDROPS:
5575 txr = adapter->tx_rings;
5576 for (int i = 0; i < adapter->num_queues; i++, txr++)
5577 rv += txr->br->br_drops;
5581 return (if_get_counter_default(ifp, cnt));
5586 igb_get_counter(if_t ifp, ift_counter cnt)
5588 struct adapter *adapter;
5589 struct e1000_hw_stats *stats;
5590 #ifndef IGB_LEGACY_TX
5591 struct tx_ring *txr;
5595 adapter = if_getsoftc(ifp);
5596 if (adapter->vf_ifp)
5597 return (igb_get_vf_counter(ifp, cnt));
5599 stats = (struct e1000_hw_stats *)adapter->stats;
5602 case IFCOUNTER_IPACKETS:
5603 return (stats->gprc);
5604 case IFCOUNTER_OPACKETS:
5605 return (stats->gptc);
5606 case IFCOUNTER_IBYTES:
5607 return (stats->gorc);
5608 case IFCOUNTER_OBYTES:
5609 return (stats->gotc);
5610 case IFCOUNTER_IMCASTS:
5611 return (stats->mprc);
5612 case IFCOUNTER_OMCASTS:
5613 return (stats->mptc);
5614 case IFCOUNTER_IERRORS:
5615 return (adapter->dropped_pkts + stats->rxerrc +
5616 stats->crcerrs + stats->algnerrc +
5617 stats->ruc + stats->roc + stats->cexterr);
5618 case IFCOUNTER_OERRORS:
5619 return (stats->ecol + stats->latecol +
5620 adapter->watchdog_events);
5621 case IFCOUNTER_COLLISIONS:
5622 return (stats->colc);
5623 case IFCOUNTER_IQDROPS:
5624 return (stats->mpc);
5625 #ifndef IGB_LEGACY_TX
5626 case IFCOUNTER_OQDROPS:
5628 txr = adapter->tx_rings;
5629 for (int i = 0; i < adapter->num_queues; i++, txr++)
5630 rv += txr->br->br_drops;
5634 return (if_get_counter_default(ifp, cnt));
5638 /**********************************************************************
5640 * Update the board statistics counters.
5642 **********************************************************************/
5644 igb_update_stats_counters(struct adapter *adapter)
5646 struct e1000_hw *hw = &adapter->hw;
5647 struct e1000_hw_stats *stats;
5650 ** The virtual function adapter has only a
5651 ** small controlled set of stats, do only
5652 ** those and return.
5654 if (adapter->vf_ifp) {
5655 igb_update_vf_stats_counters(adapter);
5659 stats = (struct e1000_hw_stats *)adapter->stats;
5661 if (adapter->hw.phy.media_type == e1000_media_type_copper ||
5662 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
5664 E1000_READ_REG(hw,E1000_SYMERRS);
5665 stats->sec += E1000_READ_REG(hw, E1000_SEC);
5668 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5669 stats->mpc += E1000_READ_REG(hw, E1000_MPC);
5670 stats->scc += E1000_READ_REG(hw, E1000_SCC);
5671 stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
5673 stats->mcc += E1000_READ_REG(hw, E1000_MCC);
5674 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
5675 stats->colc += E1000_READ_REG(hw, E1000_COLC);
5676 stats->dc += E1000_READ_REG(hw, E1000_DC);
5677 stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
5678 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5679 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5681 ** For watchdog management we need to know if we have been
5682 ** paused during the last interval, so capture that here.
5684 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5685 stats->xoffrxc += adapter->pause_frames;
5686 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5687 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5688 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
5689 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
5690 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
5691 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
5692 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5693 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5694 stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
5695 stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
5696 stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
5697 stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
5699 /* For the 64-bit byte counters the low dword must be read first. */
5700 /* Both registers clear on the read of the high dword */
5702 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
5703 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
5704 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
5705 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
5707 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
5708 stats->ruc += E1000_READ_REG(hw, E1000_RUC);
5709 stats->rfc += E1000_READ_REG(hw, E1000_RFC);
5710 stats->roc += E1000_READ_REG(hw, E1000_ROC);
5711 stats->rjc += E1000_READ_REG(hw, E1000_RJC);
5713 stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5714 stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC);
5715 stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5717 stats->tor += E1000_READ_REG(hw, E1000_TORL) +
5718 ((u64)E1000_READ_REG(hw, E1000_TORH) << 32);
5719 stats->tot += E1000_READ_REG(hw, E1000_TOTL) +
5720 ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32);
5722 stats->tpr += E1000_READ_REG(hw, E1000_TPR);
5723 stats->tpt += E1000_READ_REG(hw, E1000_TPT);
5724 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5725 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5726 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5727 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5728 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5729 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5730 stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
5731 stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
5733 /* Interrupt Counts */
5735 stats->iac += E1000_READ_REG(hw, E1000_IAC);
5736 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5737 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5738 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5739 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5740 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5741 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5742 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5743 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5745 /* Host to Card Statistics */
5747 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
5748 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
5749 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
5750 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
5751 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
5752 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
5753 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
5754 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
5755 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
5756 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
5757 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
5758 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
5759 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
5760 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
5762 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5763 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5764 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5765 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
5766 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5767 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5769 /* Driver specific counters */
5770 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
5771 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
5772 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
5773 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
5774 adapter->packet_buf_alloc_tx =
5775 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
5776 adapter->packet_buf_alloc_rx =
5777 (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
5781 /**********************************************************************
5783 * Initialize the VF board statistics counters.
5785 **********************************************************************/
5787 igb_vf_init_stats(struct adapter *adapter)
5789 struct e1000_hw *hw = &adapter->hw;
5790 struct e1000_vf_stats *stats;
5792 stats = (struct e1000_vf_stats *)adapter->stats;
5795 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
5796 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
5797 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
5798 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
5799 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
5802 /**********************************************************************
5804 * Update the VF board statistics counters.
5806 **********************************************************************/
5808 igb_update_vf_stats_counters(struct adapter *adapter)
5810 struct e1000_hw *hw = &adapter->hw;
5811 struct e1000_vf_stats *stats;
5813 if (adapter->link_speed == 0)
5816 stats = (struct e1000_vf_stats *)adapter->stats;
5818 UPDATE_VF_REG(E1000_VFGPRC,
5819 stats->last_gprc, stats->gprc);
5820 UPDATE_VF_REG(E1000_VFGORC,
5821 stats->last_gorc, stats->gorc);
5822 UPDATE_VF_REG(E1000_VFGPTC,
5823 stats->last_gptc, stats->gptc);
5824 UPDATE_VF_REG(E1000_VFGOTC,
5825 stats->last_gotc, stats->gotc);
5826 UPDATE_VF_REG(E1000_VFMPRC,
5827 stats->last_mprc, stats->mprc);
5830 /* Export a single 32-bit register via a read-only sysctl. */
5832 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5834 struct adapter *adapter;
5837 adapter = oidp->oid_arg1;
5838 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5839 return (sysctl_handle_int(oidp, &val, 0, req));
5843 ** Tuneable interrupt rate handler
5846 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5848 struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1);
5850 u32 reg, usec, rate;
5852 reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
5853 usec = ((reg & 0x7FFC) >> 2);
5855 rate = 1000000 / usec;
5858 error = sysctl_handle_int(oidp, &rate, 0, req);
5859 if (error || !req->newptr)
5865 * Add sysctl variables, one per statistic, to the system.
5868 igb_add_hw_stats(struct adapter *adapter)
5870 device_t dev = adapter->dev;
5872 struct tx_ring *txr = adapter->tx_rings;
5873 struct rx_ring *rxr = adapter->rx_rings;
5875 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5876 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5877 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5878 struct e1000_hw_stats *stats = adapter->stats;
5880 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
5881 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
5883 #define QUEUE_NAME_LEN 32
5884 char namebuf[QUEUE_NAME_LEN];
5886 /* Driver Statistics */
5887 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5888 CTLFLAG_RD, &adapter->dropped_pkts,
5889 "Driver dropped packets");
5890 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5891 CTLFLAG_RD, &adapter->link_irq,
5892 "Link MSIX IRQ Handled");
5893 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
5894 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5895 "Defragmenting mbuf chain failed");
5896 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5897 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5898 "Driver tx dma failure in xmit");
5899 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5900 CTLFLAG_RD, &adapter->rx_overruns,
5902 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5903 CTLFLAG_RD, &adapter->watchdog_events,
5904 "Watchdog timeouts");
5906 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
5907 CTLFLAG_RD, &adapter->device_control,
5908 "Device Control Register");
5909 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
5910 CTLFLAG_RD, &adapter->rx_control,
5911 "Receiver Control Register");
5912 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
5913 CTLFLAG_RD, &adapter->int_mask,
5915 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
5916 CTLFLAG_RD, &adapter->eint_mask,
5917 "Extended Interrupt Mask");
5918 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
5919 CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
5920 "Transmit Buffer Packet Allocation");
5921 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
5922 CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
5923 "Receive Buffer Packet Allocation");
5924 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5925 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5926 "Flow Control High Watermark");
5927 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5928 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5929 "Flow Control Low Watermark");
5931 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5932 struct lro_ctrl *lro = &rxr->lro;
5934 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5935 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5936 CTLFLAG_RD, NULL, "Queue Name");
5937 queue_list = SYSCTL_CHILDREN(queue_node);
5939 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5940 CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i],
5941 sizeof(&adapter->queues[i]),
5942 igb_sysctl_interrupt_rate_handler,
5943 "IU", "Interrupt Rate");
5945 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5946 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me),
5947 igb_sysctl_reg_handler, "IU",
5948 "Transmit Descriptor Head");
5949 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5950 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me),
5951 igb_sysctl_reg_handler, "IU",
5952 "Transmit Descriptor Tail");
5953 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5954 CTLFLAG_RD, &txr->no_desc_avail,
5955 "Queue Descriptors Unavailable");
5956 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5957 CTLFLAG_RD, &txr->total_packets,
5958 "Queue Packets Transmitted");
5960 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5961 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
5962 igb_sysctl_reg_handler, "IU",
5963 "Receive Descriptor Head");
5964 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5965 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
5966 igb_sysctl_reg_handler, "IU",
5967 "Receive Descriptor Tail");
5968 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5969 CTLFLAG_RD, &rxr->rx_packets,
5970 "Queue Packets Received");
5971 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5972 CTLFLAG_RD, &rxr->rx_bytes,
5973 "Queue Bytes Received");
5974 SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_queued",
5975 CTLFLAG_RD, &lro->lro_queued, 0,
5977 SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_flushed",
5978 CTLFLAG_RD, &lro->lro_flushed, 0,
5982 /* MAC stats get their own sub node */
5984 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5985 CTLFLAG_RD, NULL, "MAC Statistics");
5986 stat_list = SYSCTL_CHILDREN(stat_node);
5989 ** VF adapter has a very limited set of stats
5990 ** since its not managing the metal, so to speak.
5992 if (adapter->vf_ifp) {
5993 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5994 CTLFLAG_RD, &stats->gprc,
5995 "Good Packets Received");
5996 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5997 CTLFLAG_RD, &stats->gptc,
5998 "Good Packets Transmitted");
5999 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
6000 CTLFLAG_RD, &stats->gorc,
6001 "Good Octets Received");
6002 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
6003 CTLFLAG_RD, &stats->gotc,
6004 "Good Octets Transmitted");
6005 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
6006 CTLFLAG_RD, &stats->mprc,
6007 "Multicast Packets Received");
6011 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
6012 CTLFLAG_RD, &stats->ecol,
6013 "Excessive collisions");
6014 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
6015 CTLFLAG_RD, &stats->scc,
6016 "Single collisions");
6017 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
6018 CTLFLAG_RD, &stats->mcc,
6019 "Multiple collisions");
6020 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
6021 CTLFLAG_RD, &stats->latecol,
6023 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
6024 CTLFLAG_RD, &stats->colc,
6026 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
6027 CTLFLAG_RD, &stats->symerrs,
6029 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
6030 CTLFLAG_RD, &stats->sec,
6032 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
6033 CTLFLAG_RD, &stats->dc,
6035 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
6036 CTLFLAG_RD, &stats->mpc,
6038 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors",
6039 CTLFLAG_RD, &stats->rlec,
6040 "Receive Length Errors");
6041 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
6042 CTLFLAG_RD, &stats->rnbc,
6043 "Receive No Buffers");
6044 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
6045 CTLFLAG_RD, &stats->ruc,
6046 "Receive Undersize");
6047 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
6048 CTLFLAG_RD, &stats->rfc,
6049 "Fragmented Packets Received");
6050 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
6051 CTLFLAG_RD, &stats->roc,
6052 "Oversized Packets Received");
6053 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
6054 CTLFLAG_RD, &stats->rjc,
6056 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
6057 CTLFLAG_RD, &stats->rxerrc,
6059 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
6060 CTLFLAG_RD, &stats->crcerrs,
6062 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
6063 CTLFLAG_RD, &stats->algnerrc,
6064 "Alignment Errors");
6065 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs",
6066 CTLFLAG_RD, &stats->tncrs,
6067 "Transmit with No CRS");
6068 /* On 82575 these are collision counts */
6069 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
6070 CTLFLAG_RD, &stats->cexterr,
6071 "Collision/Carrier extension errors");
6072 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
6073 CTLFLAG_RD, &stats->xonrxc,
6075 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
6076 CTLFLAG_RD, &stats->xontxc,
6078 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
6079 CTLFLAG_RD, &stats->xoffrxc,
6081 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
6082 CTLFLAG_RD, &stats->xofftxc,
6083 "XOFF Transmitted");
6084 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd",
6085 CTLFLAG_RD, &stats->fcruc,
6086 "Unsupported Flow Control Received");
6087 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd",
6088 CTLFLAG_RD, &stats->mgprc,
6089 "Management Packets Received");
6090 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop",
6091 CTLFLAG_RD, &stats->mgpdc,
6092 "Management Packets Dropped");
6093 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd",
6094 CTLFLAG_RD, &stats->mgptc,
6095 "Management Packets Transmitted");
6096 /* Packet Reception Stats */
6097 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
6098 CTLFLAG_RD, &stats->tpr,
6099 "Total Packets Received");
6100 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
6101 CTLFLAG_RD, &stats->gprc,
6102 "Good Packets Received");
6103 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
6104 CTLFLAG_RD, &stats->bprc,
6105 "Broadcast Packets Received");
6106 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
6107 CTLFLAG_RD, &stats->mprc,
6108 "Multicast Packets Received");
6109 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
6110 CTLFLAG_RD, &stats->prc64,
6111 "64 byte frames received");
6112 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
6113 CTLFLAG_RD, &stats->prc127,
6114 "65-127 byte frames received");
6115 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
6116 CTLFLAG_RD, &stats->prc255,
6117 "128-255 byte frames received");
6118 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
6119 CTLFLAG_RD, &stats->prc511,
6120 "256-511 byte frames received");
6121 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
6122 CTLFLAG_RD, &stats->prc1023,
6123 "512-1023 byte frames received");
6124 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
6125 CTLFLAG_RD, &stats->prc1522,
6126 "1023-1522 byte frames received");
6127 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
6128 CTLFLAG_RD, &stats->gorc,
6129 "Good Octets Received");
6130 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd",
6131 CTLFLAG_RD, &stats->tor,
6132 "Total Octets Received");
6134 /* Packet Transmission Stats */
6135 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
6136 CTLFLAG_RD, &stats->gotc,
6137 "Good Octets Transmitted");
6138 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd",
6139 CTLFLAG_RD, &stats->tot,
6140 "Total Octets Transmitted");
6141 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
6142 CTLFLAG_RD, &stats->tpt,
6143 "Total Packets Transmitted");
6144 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
6145 CTLFLAG_RD, &stats->gptc,
6146 "Good Packets Transmitted");
6147 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
6148 CTLFLAG_RD, &stats->bptc,
6149 "Broadcast Packets Transmitted");
6150 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
6151 CTLFLAG_RD, &stats->mptc,
6152 "Multicast Packets Transmitted");
6153 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
6154 CTLFLAG_RD, &stats->ptc64,
6155 "64 byte frames transmitted");
6156 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
6157 CTLFLAG_RD, &stats->ptc127,
6158 "65-127 byte frames transmitted");
6159 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
6160 CTLFLAG_RD, &stats->ptc255,
6161 "128-255 byte frames transmitted");
6162 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
6163 CTLFLAG_RD, &stats->ptc511,
6164 "256-511 byte frames transmitted");
6165 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
6166 CTLFLAG_RD, &stats->ptc1023,
6167 "512-1023 byte frames transmitted");
6168 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
6169 CTLFLAG_RD, &stats->ptc1522,
6170 "1024-1522 byte frames transmitted");
6171 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
6172 CTLFLAG_RD, &stats->tsctc,
6173 "TSO Contexts Transmitted");
6174 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
6175 CTLFLAG_RD, &stats->tsctfc,
6176 "TSO Contexts Failed");
6179 /* Interrupt Stats */
6181 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
6182 CTLFLAG_RD, NULL, "Interrupt Statistics");
6183 int_list = SYSCTL_CHILDREN(int_node);
6185 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
6186 CTLFLAG_RD, &stats->iac,
6187 "Interrupt Assertion Count");
6189 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
6190 CTLFLAG_RD, &stats->icrxptc,
6191 "Interrupt Cause Rx Pkt Timer Expire Count");
6193 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
6194 CTLFLAG_RD, &stats->icrxatc,
6195 "Interrupt Cause Rx Abs Timer Expire Count");
6197 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
6198 CTLFLAG_RD, &stats->ictxptc,
6199 "Interrupt Cause Tx Pkt Timer Expire Count");
6201 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
6202 CTLFLAG_RD, &stats->ictxatc,
6203 "Interrupt Cause Tx Abs Timer Expire Count");
6205 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
6206 CTLFLAG_RD, &stats->ictxqec,
6207 "Interrupt Cause Tx Queue Empty Count");
6209 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
6210 CTLFLAG_RD, &stats->ictxqmtc,
6211 "Interrupt Cause Tx Queue Min Thresh Count");
6213 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
6214 CTLFLAG_RD, &stats->icrxdmtc,
6215 "Interrupt Cause Rx Desc Min Thresh Count");
6217 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
6218 CTLFLAG_RD, &stats->icrxoc,
6219 "Interrupt Cause Receiver Overrun Count");
6221 /* Host to Card Stats */
6223 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
6225 "Host to Card Statistics");
6227 host_list = SYSCTL_CHILDREN(host_node);
6229 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
6230 CTLFLAG_RD, &stats->cbtmpc,
6231 "Circuit Breaker Tx Packet Count");
6233 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
6234 CTLFLAG_RD, &stats->htdpmc,
6235 "Host Transmit Discarded Packets");
6237 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
6238 CTLFLAG_RD, &stats->rpthc,
6239 "Rx Packets To Host");
6241 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
6242 CTLFLAG_RD, &stats->cbrmpc,
6243 "Circuit Breaker Rx Packet Count");
6245 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
6246 CTLFLAG_RD, &stats->cbrdpc,
6247 "Circuit Breaker Rx Dropped Count");
6249 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
6250 CTLFLAG_RD, &stats->hgptc,
6251 "Host Good Packets Tx Count");
6253 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
6254 CTLFLAG_RD, &stats->htcbdpc,
6255 "Host Tx Circuit Breaker Dropped Count");
6257 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
6258 CTLFLAG_RD, &stats->hgorc,
6259 "Host Good Octets Received Count");
6261 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
6262 CTLFLAG_RD, &stats->hgotc,
6263 "Host Good Octets Transmit Count");
6265 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
6266 CTLFLAG_RD, &stats->lenerrs,
6269 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
6270 CTLFLAG_RD, &stats->scvpc,
6271 "SerDes/SGMII Code Violation Pkt Count");
6273 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
6274 CTLFLAG_RD, &stats->hrmpc,
6275 "Header Redirection Missed Packet Count");
6279 /**********************************************************************
6281 * This routine provides a way to dump out the adapter eeprom,
6282 * often a useful debug/service tool. This only dumps the first
6283 * 32 words, stuff that matters is in that extent.
6285 **********************************************************************/
6287 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
6289 struct adapter *adapter;
6294 error = sysctl_handle_int(oidp, &result, 0, req);
6296 if (error || !req->newptr)
6300 * This value will cause a hex dump of the
6301 * first 32 16-bit words of the EEPROM to
6305 adapter = (struct adapter *)arg1;
6306 igb_print_nvm_info(adapter);
6313 igb_print_nvm_info(struct adapter *adapter)
6318 /* Its a bit crude, but it gets the job done */
6319 printf("\nInterface EEPROM Dump:\n");
6320 printf("Offset\n0x0000 ");
6321 for (i = 0, j = 0; i < 32; i++, j++) {
6322 if (j == 8) { /* Make the offset block */
6324 printf("\n0x00%x0 ",row);
6326 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
6327 printf("%04x ", eeprom_data);
6333 igb_set_sysctl_value(struct adapter *adapter, const char *name,
6334 const char *description, int *limit, int value)
6337 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
6338 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
6339 OID_AUTO, name, CTLFLAG_RW, limit, value, description);
6343 ** Set flow control using sysctl:
6344 ** Flow control values:
6351 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
6354 static int input = 3; /* default is full */
6355 struct adapter *adapter = (struct adapter *) arg1;
6357 error = sysctl_handle_int(oidp, &input, 0, req);
6359 if ((error) || (req->newptr == NULL))
6363 case e1000_fc_rx_pause:
6364 case e1000_fc_tx_pause:
6367 adapter->hw.fc.requested_mode = input;
6368 adapter->fc = input;
6375 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
6376 e1000_force_mac_fc(&adapter->hw);
6377 /* XXX TODO: update DROP_EN on each RX queue if appropriate */
6382 ** Manage DMA Coalesce:
6385 ** Legal timer values are:
6386 ** 250,500,1000-10000 in thousands
6389 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
6391 struct adapter *adapter = (struct adapter *) arg1;
6394 error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
6396 if ((error) || (req->newptr == NULL))
6399 switch (adapter->dmac) {
6403 case 1: /* Just enable and use default */
6404 adapter->dmac = 1000;
6418 /* Legal values - allow */
6421 /* Do nothing, illegal value */
6425 /* Reinit the interface */
6431 ** Manage Energy Efficient Ethernet:
6433 ** 0/1 - enabled/disabled
6436 igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
6438 struct adapter *adapter = (struct adapter *) arg1;
6441 value = adapter->hw.dev_spec._82575.eee_disable;
6442 error = sysctl_handle_int(oidp, &value, 0, req);
6443 if (error || req->newptr == NULL)
6445 IGB_CORE_LOCK(adapter);
6446 adapter->hw.dev_spec._82575.eee_disable = (value != 0);
6447 igb_init_locked(adapter);
6448 IGB_CORE_UNLOCK(adapter);