1 /******************************************************************************
3 Copyright (c) 2001-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
40 #ifdef HAVE_KERNEL_OPTION_HEADERS
41 #include "opt_device_polling.h"
47 /*********************************************************************
49 *********************************************************************/
50 char igb_driver_version[] = "2.5.3-k";
53 /*********************************************************************
56 * Used by probe to select devices to load on
57 * Last field stores an index into e1000_strings
58 * Last entry must be all 0s
60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61 *********************************************************************/
63 static igb_vendor_info_t igb_vendor_info_array[] =
65 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0},
66 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0},
67 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0},
68 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0},
69 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0},
70 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0},
71 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0},
72 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0},
73 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0},
74 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0},
75 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0},
76 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0},
77 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0},
78 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0},
79 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0},
80 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0},
81 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0},
82 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0},
83 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0},
84 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0},
85 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0},
86 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0},
87 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0},
88 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0},
89 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0},
90 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0},
91 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0},
92 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0},
93 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0},
94 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0},
95 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0},
96 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0},
97 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0},
98 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0},
99 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0},
100 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0},
101 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0},
102 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0},
103 {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0},
104 /* required last entry */
108 /*********************************************************************
109 * Table of branding strings for all supported NICs.
110 *********************************************************************/
112 static char *igb_strings[] = {
113 "Intel(R) PRO/1000 Network Connection"
116 /*********************************************************************
117 * Function prototypes
118 *********************************************************************/
119 static int igb_probe(device_t);
120 static int igb_attach(device_t);
121 static int igb_detach(device_t);
122 static int igb_shutdown(device_t);
123 static int igb_suspend(device_t);
124 static int igb_resume(device_t);
125 #ifndef IGB_LEGACY_TX
126 static int igb_mq_start(struct ifnet *, struct mbuf *);
127 static int igb_mq_start_locked(struct ifnet *, struct tx_ring *);
128 static void igb_qflush(struct ifnet *);
129 static void igb_deferred_mq_start(void *, int);
131 static void igb_start(struct ifnet *);
132 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp);
134 static int igb_ioctl(struct ifnet *, u_long, caddr_t);
135 static uint64_t igb_get_counter(if_t, ift_counter);
136 static void igb_init(void *);
137 static void igb_init_locked(struct adapter *);
138 static void igb_stop(void *);
139 static void igb_media_status(struct ifnet *, struct ifmediareq *);
140 static int igb_media_change(struct ifnet *);
141 static void igb_identify_hardware(struct adapter *);
142 static int igb_allocate_pci_resources(struct adapter *);
143 static int igb_allocate_msix(struct adapter *);
144 static int igb_allocate_legacy(struct adapter *);
145 static int igb_setup_msix(struct adapter *);
146 static void igb_free_pci_resources(struct adapter *);
147 static void igb_local_timer(void *);
148 static void igb_reset(struct adapter *);
149 static int igb_setup_interface(device_t, struct adapter *);
150 static int igb_allocate_queues(struct adapter *);
151 static void igb_configure_queues(struct adapter *);
153 static int igb_allocate_transmit_buffers(struct tx_ring *);
154 static void igb_setup_transmit_structures(struct adapter *);
155 static void igb_setup_transmit_ring(struct tx_ring *);
156 static void igb_initialize_transmit_units(struct adapter *);
157 static void igb_free_transmit_structures(struct adapter *);
158 static void igb_free_transmit_buffers(struct tx_ring *);
160 static int igb_allocate_receive_buffers(struct rx_ring *);
161 static int igb_setup_receive_structures(struct adapter *);
162 static int igb_setup_receive_ring(struct rx_ring *);
163 static void igb_initialize_receive_units(struct adapter *);
164 static void igb_free_receive_structures(struct adapter *);
165 static void igb_free_receive_buffers(struct rx_ring *);
166 static void igb_free_receive_ring(struct rx_ring *);
168 static void igb_enable_intr(struct adapter *);
169 static void igb_disable_intr(struct adapter *);
170 static void igb_update_stats_counters(struct adapter *);
171 static bool igb_txeof(struct tx_ring *);
173 static __inline void igb_rx_discard(struct rx_ring *, int);
174 static __inline void igb_rx_input(struct rx_ring *,
175 struct ifnet *, struct mbuf *, u32);
177 static bool igb_rxeof(struct igb_queue *, int, int *);
178 static void igb_rx_checksum(u32, struct mbuf *, u32);
179 static int igb_tx_ctx_setup(struct tx_ring *,
180 struct mbuf *, u32 *, u32 *);
181 static int igb_tso_setup(struct tx_ring *,
182 struct mbuf *, u32 *, u32 *);
183 static void igb_set_promisc(struct adapter *);
184 static void igb_disable_promisc(struct adapter *);
185 static void igb_set_multi(struct adapter *);
186 static void igb_update_link_status(struct adapter *);
187 static void igb_refresh_mbufs(struct rx_ring *, int);
189 static void igb_register_vlan(void *, struct ifnet *, u16);
190 static void igb_unregister_vlan(void *, struct ifnet *, u16);
191 static void igb_setup_vlan_hw_support(struct adapter *);
193 static int igb_xmit(struct tx_ring *, struct mbuf **);
194 static int igb_dma_malloc(struct adapter *, bus_size_t,
195 struct igb_dma_alloc *, int);
196 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *);
197 static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
198 static void igb_print_nvm_info(struct adapter *);
199 static int igb_is_valid_ether_addr(u8 *);
200 static void igb_add_hw_stats(struct adapter *);
202 static void igb_vf_init_stats(struct adapter *);
203 static void igb_update_vf_stats_counters(struct adapter *);
205 /* Management and WOL Support */
206 static void igb_init_manageability(struct adapter *);
207 static void igb_release_manageability(struct adapter *);
208 static void igb_get_hw_control(struct adapter *);
209 static void igb_release_hw_control(struct adapter *);
210 static void igb_enable_wakeup(device_t);
211 static void igb_led_func(void *, int);
213 static int igb_irq_fast(void *);
214 static void igb_msix_que(void *);
215 static void igb_msix_link(void *);
216 static void igb_handle_que(void *context, int pending);
217 static void igb_handle_link(void *context, int pending);
218 static void igb_handle_link_locked(struct adapter *);
220 static void igb_set_sysctl_value(struct adapter *, const char *,
221 const char *, int *, int);
222 static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
223 static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
224 static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
226 #ifdef DEVICE_POLLING
227 static poll_handler_t igb_poll;
230 /*********************************************************************
231 * FreeBSD Device Interface Entry Points
232 *********************************************************************/
234 static device_method_t igb_methods[] = {
235 /* Device interface */
236 DEVMETHOD(device_probe, igb_probe),
237 DEVMETHOD(device_attach, igb_attach),
238 DEVMETHOD(device_detach, igb_detach),
239 DEVMETHOD(device_shutdown, igb_shutdown),
240 DEVMETHOD(device_suspend, igb_suspend),
241 DEVMETHOD(device_resume, igb_resume),
245 static driver_t igb_driver = {
246 "igb", igb_methods, sizeof(struct adapter),
249 static devclass_t igb_devclass;
250 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
251 MODULE_DEPEND(igb, pci, 1, 1, 1);
252 MODULE_DEPEND(igb, ether, 1, 1, 1);
254 MODULE_DEPEND(igb, netmap, 1, 1, 1);
255 #endif /* DEV_NETMAP */
257 /*********************************************************************
258 * Tunable default values.
259 *********************************************************************/
261 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
263 /* Descriptor defaults */
264 static int igb_rxd = IGB_DEFAULT_RXD;
265 static int igb_txd = IGB_DEFAULT_TXD;
266 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
267 "Number of receive descriptors per queue");
268 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
269 "Number of transmit descriptors per queue");
272 ** AIM: Adaptive Interrupt Moderation
273 ** which means that the interrupt rate
274 ** is varied over time based on the
275 ** traffic for that interrupt vector
277 static int igb_enable_aim = TRUE;
278 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0,
279 "Enable adaptive interrupt moderation");
282 * MSIX should be the default for best performance,
283 * but this allows it to be forced off for testing.
285 static int igb_enable_msix = 1;
286 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
287 "Enable MSI-X interrupts");
290 ** Tuneable Interrupt rate
292 static int igb_max_interrupt_rate = 8000;
293 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
294 &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
296 #ifndef IGB_LEGACY_TX
298 ** Tuneable number of buffers in the buf-ring (drbr_xxx)
300 static int igb_buf_ring_size = IGB_BR_SIZE;
301 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
302 &igb_buf_ring_size, 0, "Size of the bufring");
306 ** Header split causes the packet header to
307 ** be dma'd to a separate mbuf from the payload.
308 ** this can have memory alignment benefits. But
309 ** another plus is that small packets often fit
310 ** into the header and thus use no cluster. Its
311 ** a very workload dependent type feature.
313 static int igb_header_split = FALSE;
314 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
315 "Enable receive mbuf header split");
318 ** This will autoconfigure based on the
319 ** number of CPUs and max supported
320 ** MSIX messages if left at 0.
322 static int igb_num_queues = 0;
323 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
324 "Number of queues to configure, 0 indicates autoconfigure");
327 ** Global variable to store last used CPU when binding queues
328 ** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
329 ** queue is bound to a cpu.
331 static int igb_last_bind_cpu = -1;
333 /* How many packets rxeof tries to clean at a time */
334 static int igb_rx_process_limit = 100;
335 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
336 &igb_rx_process_limit, 0,
337 "Maximum number of received packets to process at a time, -1 means unlimited");
339 /* How many packets txeof tries to clean at a time */
340 static int igb_tx_process_limit = -1;
341 SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
342 &igb_tx_process_limit, 0,
343 "Maximum number of sent packets to process at a time, -1 means unlimited");
345 #ifdef DEV_NETMAP /* see ixgbe.c for details */
346 #include <dev/netmap/if_igb_netmap.h>
347 #endif /* DEV_NETMAP */
348 /*********************************************************************
349 * Device identification routine
351 * igb_probe determines if the driver should be loaded on
352 * adapter based on PCI vendor/device id of the adapter.
354 * return BUS_PROBE_DEFAULT on success, positive on failure
355 *********************************************************************/
358 igb_probe(device_t dev)
360 char adapter_name[256];
361 uint16_t pci_vendor_id = 0;
362 uint16_t pci_device_id = 0;
363 uint16_t pci_subvendor_id = 0;
364 uint16_t pci_subdevice_id = 0;
365 igb_vendor_info_t *ent;
367 INIT_DEBUGOUT("igb_probe: begin");
369 pci_vendor_id = pci_get_vendor(dev);
370 if (pci_vendor_id != IGB_INTEL_VENDOR_ID)
373 pci_device_id = pci_get_device(dev);
374 pci_subvendor_id = pci_get_subvendor(dev);
375 pci_subdevice_id = pci_get_subdevice(dev);
377 ent = igb_vendor_info_array;
378 while (ent->vendor_id != 0) {
379 if ((pci_vendor_id == ent->vendor_id) &&
380 (pci_device_id == ent->device_id) &&
382 ((pci_subvendor_id == ent->subvendor_id) ||
383 (ent->subvendor_id == 0)) &&
385 ((pci_subdevice_id == ent->subdevice_id) ||
386 (ent->subdevice_id == 0))) {
387 sprintf(adapter_name, "%s, Version - %s",
388 igb_strings[ent->index],
390 device_set_desc_copy(dev, adapter_name);
391 return (BUS_PROBE_DEFAULT);
398 /*********************************************************************
399 * Device initialization routine
401 * The attach entry point is called when the driver is being loaded.
402 * This routine identifies the type of hardware, allocates all resources
403 * and initializes the hardware.
405 * return 0 on success, positive on failure
406 *********************************************************************/
409 igb_attach(device_t dev)
411 struct adapter *adapter;
415 INIT_DEBUGOUT("igb_attach: begin");
417 if (resource_disabled("igb", device_get_unit(dev))) {
418 device_printf(dev, "Disabled by device hint\n");
422 adapter = device_get_softc(dev);
423 adapter->dev = adapter->osdep.dev = dev;
424 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
427 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
428 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
429 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
430 igb_sysctl_nvm_info, "I", "NVM Information");
432 igb_set_sysctl_value(adapter, "enable_aim",
433 "Interrupt Moderation", &adapter->enable_aim,
436 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
437 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
438 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
439 adapter, 0, igb_set_flowcntl, "I", "Flow Control");
441 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
443 /* Determine hardware and mac info */
444 igb_identify_hardware(adapter);
446 /* Setup PCI resources */
447 if (igb_allocate_pci_resources(adapter)) {
448 device_printf(dev, "Allocation of PCI resources failed\n");
453 /* Do Shared Code initialization */
454 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
455 device_printf(dev, "Setup of Shared code failed\n");
460 e1000_get_bus_info(&adapter->hw);
462 /* Sysctls for limiting the amount of work done in the taskqueues */
463 igb_set_sysctl_value(adapter, "rx_processing_limit",
464 "max number of rx packets to process",
465 &adapter->rx_process_limit, igb_rx_process_limit);
467 igb_set_sysctl_value(adapter, "tx_processing_limit",
468 "max number of tx packets to process",
469 &adapter->tx_process_limit, igb_tx_process_limit);
472 * Validate number of transmit and receive descriptors. It
473 * must not exceed hardware maximum, and must be multiple
474 * of E1000_DBA_ALIGN.
476 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
477 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
478 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
479 IGB_DEFAULT_TXD, igb_txd);
480 adapter->num_tx_desc = IGB_DEFAULT_TXD;
482 adapter->num_tx_desc = igb_txd;
483 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
484 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
485 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
486 IGB_DEFAULT_RXD, igb_rxd);
487 adapter->num_rx_desc = IGB_DEFAULT_RXD;
489 adapter->num_rx_desc = igb_rxd;
491 adapter->hw.mac.autoneg = DO_AUTO_NEG;
492 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
493 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
496 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
497 adapter->hw.phy.mdix = AUTO_ALL_MODES;
498 adapter->hw.phy.disable_polarity_correction = FALSE;
499 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
503 * Set the frame limits assuming
504 * standard ethernet sized frames.
506 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
509 ** Allocate and Setup Queues
511 if (igb_allocate_queues(adapter)) {
516 /* Allocate the appropriate stats memory */
517 if (adapter->vf_ifp) {
519 (struct e1000_vf_stats *)malloc(sizeof \
520 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
521 igb_vf_init_stats(adapter);
524 (struct e1000_hw_stats *)malloc(sizeof \
525 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
526 if (adapter->stats == NULL) {
527 device_printf(dev, "Can not allocate stats memory\n");
532 /* Allocate multicast array memory. */
533 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
534 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
535 if (adapter->mta == NULL) {
536 device_printf(dev, "Can not allocate multicast setup array\n");
541 /* Some adapter-specific advanced features */
542 if (adapter->hw.mac.type >= e1000_i350) {
543 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
544 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
545 OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
546 adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
547 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
548 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
549 OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
550 adapter, 0, igb_sysctl_eee, "I",
551 "Disable Energy Efficient Ethernet");
552 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
553 if (adapter->hw.mac.type == e1000_i354)
554 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
556 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
561 ** Start from a known state, this is
562 ** important in reading the nvm and
565 e1000_reset_hw(&adapter->hw);
567 /* Make sure we have a good EEPROM before we read from it */
568 if (((adapter->hw.mac.type != e1000_i210) &&
569 (adapter->hw.mac.type != e1000_i211)) &&
570 (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
572 ** Some PCI-E parts fail the first check due to
573 ** the link being in sleep state, call it again,
574 ** if it fails a second time its a real issue.
576 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
578 "The EEPROM Checksum Is Not Valid\n");
585 ** Copy the permanent MAC address out of the EEPROM
587 if (e1000_read_mac_addr(&adapter->hw) < 0) {
588 device_printf(dev, "EEPROM read error while reading MAC"
593 /* Check its sanity */
594 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
595 device_printf(dev, "Invalid MAC address\n");
600 /* Setup OS specific network interface */
601 if (igb_setup_interface(dev, adapter) != 0)
604 /* Now get a good starting state */
607 /* Initialize statistics */
608 igb_update_stats_counters(adapter);
610 adapter->hw.mac.get_link_status = 1;
611 igb_update_link_status(adapter);
613 /* Indicate SOL/IDER usage */
614 if (e1000_check_reset_block(&adapter->hw))
616 "PHY reset is blocked due to SOL/IDER session.\n");
618 /* Determine if we have to control management hardware */
619 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
624 /* APME bit in EEPROM is mapped to WUC.APME */
625 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
627 adapter->wol = E1000_WUFC_MAG;
629 /* Register for VLAN events */
630 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
631 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
632 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
633 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
635 igb_add_hw_stats(adapter);
637 /* Tell the stack that the interface is not active */
638 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
639 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
641 adapter->led_dev = led_create(igb_led_func, adapter,
642 device_get_nameunit(dev));
645 ** Configure Interrupts
647 if ((adapter->msix > 1) && (igb_enable_msix))
648 error = igb_allocate_msix(adapter);
649 else /* MSI or Legacy */
650 error = igb_allocate_legacy(adapter);
655 igb_netmap_attach(adapter);
656 #endif /* DEV_NETMAP */
657 INIT_DEBUGOUT("igb_attach: end");
662 if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */
664 igb_free_transmit_structures(adapter);
665 igb_free_receive_structures(adapter);
666 igb_release_hw_control(adapter);
668 igb_free_pci_resources(adapter);
669 if (adapter->ifp != NULL)
670 if_free(adapter->ifp);
671 free(adapter->mta, M_DEVBUF);
672 IGB_CORE_LOCK_DESTROY(adapter);
677 /*********************************************************************
678 * Device removal routine
680 * The detach entry point is called when the driver is being removed.
681 * This routine stops the adapter and deallocates all the resources
682 * that were allocated for driver operation.
684 * return 0 on success, positive on failure
685 *********************************************************************/
688 igb_detach(device_t dev)
690 struct adapter *adapter = device_get_softc(dev);
691 struct ifnet *ifp = adapter->ifp;
693 INIT_DEBUGOUT("igb_detach: begin");
695 /* Make sure VLANS are not using driver */
696 if (adapter->ifp->if_vlantrunk != NULL) {
697 device_printf(dev,"Vlan in use, detach first\n");
701 ether_ifdetach(adapter->ifp);
703 if (adapter->led_dev != NULL)
704 led_destroy(adapter->led_dev);
706 #ifdef DEVICE_POLLING
707 if (ifp->if_capenable & IFCAP_POLLING)
708 ether_poll_deregister(ifp);
711 IGB_CORE_LOCK(adapter);
712 adapter->in_detach = 1;
714 IGB_CORE_UNLOCK(adapter);
716 e1000_phy_hw_reset(&adapter->hw);
718 /* Give control back to firmware */
719 igb_release_manageability(adapter);
720 igb_release_hw_control(adapter);
723 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
724 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
725 igb_enable_wakeup(dev);
728 /* Unregister VLAN events */
729 if (adapter->vlan_attach != NULL)
730 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
731 if (adapter->vlan_detach != NULL)
732 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
734 callout_drain(&adapter->timer);
737 netmap_detach(adapter->ifp);
738 #endif /* DEV_NETMAP */
739 igb_free_pci_resources(adapter);
740 bus_generic_detach(dev);
743 igb_free_transmit_structures(adapter);
744 igb_free_receive_structures(adapter);
745 if (adapter->mta != NULL)
746 free(adapter->mta, M_DEVBUF);
748 IGB_CORE_LOCK_DESTROY(adapter);
753 /*********************************************************************
755 * Shutdown entry point
757 **********************************************************************/
760 igb_shutdown(device_t dev)
762 return igb_suspend(dev);
766 * Suspend/resume device methods.
769 igb_suspend(device_t dev)
771 struct adapter *adapter = device_get_softc(dev);
773 IGB_CORE_LOCK(adapter);
777 igb_release_manageability(adapter);
778 igb_release_hw_control(adapter);
781 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
782 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
783 igb_enable_wakeup(dev);
786 IGB_CORE_UNLOCK(adapter);
788 return bus_generic_suspend(dev);
792 igb_resume(device_t dev)
794 struct adapter *adapter = device_get_softc(dev);
795 struct tx_ring *txr = adapter->tx_rings;
796 struct ifnet *ifp = adapter->ifp;
798 IGB_CORE_LOCK(adapter);
799 igb_init_locked(adapter);
800 igb_init_manageability(adapter);
802 if ((ifp->if_flags & IFF_UP) &&
803 (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
804 for (int i = 0; i < adapter->num_queues; i++, txr++) {
806 #ifndef IGB_LEGACY_TX
807 /* Process the stack queue only if not depleted */
808 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
809 !drbr_empty(ifp, txr->br))
810 igb_mq_start_locked(ifp, txr);
812 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
813 igb_start_locked(txr, ifp);
818 IGB_CORE_UNLOCK(adapter);
820 return bus_generic_resume(dev);
826 /*********************************************************************
827 * Transmit entry point
829 * igb_start is called by the stack to initiate a transmit.
830 * The driver will remain in this routine as long as there are
831 * packets to transmit and transmit resources are available.
832 * In case resources are not available stack is notified and
833 * the packet is requeued.
834 **********************************************************************/
837 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
839 struct adapter *adapter = ifp->if_softc;
842 IGB_TX_LOCK_ASSERT(txr);
844 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
847 if (!adapter->link_active)
850 /* Call cleanup if number of TX descriptors low */
851 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
854 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
855 if (txr->tx_avail <= IGB_MAX_SCATTER) {
856 txr->queue_status |= IGB_QUEUE_DEPLETED;
859 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
863 * Encapsulation can modify our pointer, and or make it
864 * NULL on failure. In that event, we can't requeue.
866 if (igb_xmit(txr, &m_head)) {
868 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
869 if (txr->tx_avail <= IGB_MAX_SCATTER)
870 txr->queue_status |= IGB_QUEUE_DEPLETED;
874 /* Send a copy of the frame to the BPF listener */
875 ETHER_BPF_MTAP(ifp, m_head);
877 /* Set watchdog on */
878 txr->watchdog_time = ticks;
879 txr->queue_status |= IGB_QUEUE_WORKING;
884 * Legacy TX driver routine, called from the
885 * stack, always uses tx[0], and spins for it.
886 * Should not be used with multiqueue tx
889 igb_start(struct ifnet *ifp)
891 struct adapter *adapter = ifp->if_softc;
892 struct tx_ring *txr = adapter->tx_rings;
894 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
896 igb_start_locked(txr, ifp);
902 #else /* ~IGB_LEGACY_TX */
905 ** Multiqueue Transmit Entry:
906 ** quick turnaround to the stack
910 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
912 struct adapter *adapter = ifp->if_softc;
913 struct igb_queue *que;
920 /* Which queue to use */
922 * When doing RSS, map it to the same outbound queue
923 * as the incoming flow would be mapped to.
925 * If everything is setup correctly, it should be the
926 * same bucket that the current CPU we're on is.
928 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
930 if (rss_hash2bucket(m->m_pkthdr.flowid,
931 M_HASHTYPE_GET(m), &bucket_id) == 0) {
932 /* XXX TODO: spit out something if bucket_id > num_queues? */
933 i = bucket_id % adapter->num_queues;
936 i = m->m_pkthdr.flowid % adapter->num_queues;
941 i = curcpu % adapter->num_queues;
943 txr = &adapter->tx_rings[i];
944 que = &adapter->queues[i];
946 err = drbr_enqueue(ifp, txr->br, m);
949 if (IGB_TX_TRYLOCK(txr)) {
950 igb_mq_start_locked(ifp, txr);
953 taskqueue_enqueue(que->tq, &txr->txq_task);
959 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
961 struct adapter *adapter = txr->adapter;
963 int err = 0, enq = 0;
965 IGB_TX_LOCK_ASSERT(txr);
967 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
968 adapter->link_active == 0)
971 /* Process the queue */
972 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
973 if ((err = igb_xmit(txr, &next)) != 0) {
975 /* It was freed, move forward */
976 drbr_advance(ifp, txr->br);
979 * Still have one left, it may not be
980 * the same since the transmit function
981 * may have changed it.
983 drbr_putback(ifp, txr->br, next);
987 drbr_advance(ifp, txr->br);
989 if (next->m_flags & M_MCAST && adapter->vf_ifp)
990 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
991 ETHER_BPF_MTAP(ifp, next);
992 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
996 /* Set the watchdog */
997 txr->queue_status |= IGB_QUEUE_WORKING;
998 txr->watchdog_time = ticks;
1000 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
1002 if (txr->tx_avail <= IGB_MAX_SCATTER)
1003 txr->queue_status |= IGB_QUEUE_DEPLETED;
1008 * Called from a taskqueue to drain queued transmit packets.
1011 igb_deferred_mq_start(void *arg, int pending)
1013 struct tx_ring *txr = arg;
1014 struct adapter *adapter = txr->adapter;
1015 struct ifnet *ifp = adapter->ifp;
1018 if (!drbr_empty(ifp, txr->br))
1019 igb_mq_start_locked(ifp, txr);
1024 ** Flush all ring buffers
1027 igb_qflush(struct ifnet *ifp)
1029 struct adapter *adapter = ifp->if_softc;
1030 struct tx_ring *txr = adapter->tx_rings;
1033 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1035 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1041 #endif /* ~IGB_LEGACY_TX */
1043 /*********************************************************************
1046 * igb_ioctl is called when the user wants to configure the
1049 * return 0 on success, positive on failure
1050 **********************************************************************/
1053 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1055 struct adapter *adapter = ifp->if_softc;
1056 struct ifreq *ifr = (struct ifreq *)data;
1057 #if defined(INET) || defined(INET6)
1058 struct ifaddr *ifa = (struct ifaddr *)data;
1060 bool avoid_reset = FALSE;
1063 if (adapter->in_detach)
1069 if (ifa->ifa_addr->sa_family == AF_INET)
1073 if (ifa->ifa_addr->sa_family == AF_INET6)
1077 ** Calling init results in link renegotiation,
1078 ** so we avoid doing it when possible.
1081 ifp->if_flags |= IFF_UP;
1082 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1085 if (!(ifp->if_flags & IFF_NOARP))
1086 arp_ifinit(ifp, ifa);
1089 error = ether_ioctl(ifp, command, data);
1095 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1097 IGB_CORE_LOCK(adapter);
1098 max_frame_size = 9234;
1099 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1101 IGB_CORE_UNLOCK(adapter);
1106 ifp->if_mtu = ifr->ifr_mtu;
1107 adapter->max_frame_size =
1108 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1109 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1110 igb_init_locked(adapter);
1111 IGB_CORE_UNLOCK(adapter);
1115 IOCTL_DEBUGOUT("ioctl rcv'd:\
1116 SIOCSIFFLAGS (Set Interface Flags)");
1117 IGB_CORE_LOCK(adapter);
1118 if (ifp->if_flags & IFF_UP) {
1119 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1120 if ((ifp->if_flags ^ adapter->if_flags) &
1121 (IFF_PROMISC | IFF_ALLMULTI)) {
1122 igb_disable_promisc(adapter);
1123 igb_set_promisc(adapter);
1126 igb_init_locked(adapter);
1128 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1130 adapter->if_flags = ifp->if_flags;
1131 IGB_CORE_UNLOCK(adapter);
1135 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1136 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1137 IGB_CORE_LOCK(adapter);
1138 igb_disable_intr(adapter);
1139 igb_set_multi(adapter);
1140 #ifdef DEVICE_POLLING
1141 if (!(ifp->if_capenable & IFCAP_POLLING))
1143 igb_enable_intr(adapter);
1144 IGB_CORE_UNLOCK(adapter);
1148 /* Check SOL/IDER usage */
1149 IGB_CORE_LOCK(adapter);
1150 if (e1000_check_reset_block(&adapter->hw)) {
1151 IGB_CORE_UNLOCK(adapter);
1152 device_printf(adapter->dev, "Media change is"
1153 " blocked due to SOL/IDER session.\n");
1156 IGB_CORE_UNLOCK(adapter);
1158 IOCTL_DEBUGOUT("ioctl rcv'd: \
1159 SIOCxIFMEDIA (Get/Set Interface Media)");
1160 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1166 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1168 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1169 #ifdef DEVICE_POLLING
1170 if (mask & IFCAP_POLLING) {
1171 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1172 error = ether_poll_register(igb_poll, ifp);
1175 IGB_CORE_LOCK(adapter);
1176 igb_disable_intr(adapter);
1177 ifp->if_capenable |= IFCAP_POLLING;
1178 IGB_CORE_UNLOCK(adapter);
1180 error = ether_poll_deregister(ifp);
1181 /* Enable interrupt even in error case */
1182 IGB_CORE_LOCK(adapter);
1183 igb_enable_intr(adapter);
1184 ifp->if_capenable &= ~IFCAP_POLLING;
1185 IGB_CORE_UNLOCK(adapter);
1189 #if __FreeBSD_version >= 1000000
1190 /* HW cannot turn these on/off separately */
1191 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
1192 ifp->if_capenable ^= IFCAP_RXCSUM;
1193 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1196 if (mask & IFCAP_TXCSUM) {
1197 ifp->if_capenable ^= IFCAP_TXCSUM;
1200 if (mask & IFCAP_TXCSUM_IPV6) {
1201 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1205 if (mask & IFCAP_HWCSUM) {
1206 ifp->if_capenable ^= IFCAP_HWCSUM;
1210 if (mask & IFCAP_TSO4) {
1211 ifp->if_capenable ^= IFCAP_TSO4;
1214 if (mask & IFCAP_TSO6) {
1215 ifp->if_capenable ^= IFCAP_TSO6;
1218 if (mask & IFCAP_VLAN_HWTAGGING) {
1219 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1222 if (mask & IFCAP_VLAN_HWFILTER) {
1223 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1226 if (mask & IFCAP_VLAN_HWTSO) {
1227 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1230 if (mask & IFCAP_LRO) {
1231 ifp->if_capenable ^= IFCAP_LRO;
1234 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1236 VLAN_CAPABILITIES(ifp);
1241 error = ether_ioctl(ifp, command, data);
1249 /*********************************************************************
1252 * This routine is used in two ways. It is used by the stack as
1253 * init entry point in network interface structure. It is also used
1254 * by the driver as a hw/sw initialization routine to get to a
1257 * return 0 on success, positive on failure
1258 **********************************************************************/
1261 igb_init_locked(struct adapter *adapter)
1263 struct ifnet *ifp = adapter->ifp;
1264 device_t dev = adapter->dev;
1266 INIT_DEBUGOUT("igb_init: begin");
1268 IGB_CORE_LOCK_ASSERT(adapter);
1270 igb_disable_intr(adapter);
1271 callout_stop(&adapter->timer);
1273 /* Get the latest mac address, User can use a LAA */
1274 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1277 /* Put the address into the Receive Address Array */
1278 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1281 igb_update_link_status(adapter);
1283 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1285 /* Set hardware offload abilities */
1286 ifp->if_hwassist = 0;
1287 if (ifp->if_capenable & IFCAP_TXCSUM) {
1288 #if __FreeBSD_version >= 1000000
1289 ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP);
1290 if (adapter->hw.mac.type != e1000_82575)
1291 ifp->if_hwassist |= CSUM_IP_SCTP;
1293 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1294 #if __FreeBSD_version >= 800000
1295 if (adapter->hw.mac.type != e1000_82575)
1296 ifp->if_hwassist |= CSUM_SCTP;
1301 #if __FreeBSD_version >= 1000000
1302 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) {
1303 ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP);
1304 if (adapter->hw.mac.type != e1000_82575)
1305 ifp->if_hwassist |= CSUM_IP6_SCTP;
1308 if (ifp->if_capenable & IFCAP_TSO)
1309 ifp->if_hwassist |= CSUM_TSO;
1311 /* Clear bad data from Rx FIFOs */
1312 e1000_rx_fifo_flush_82575(&adapter->hw);
1314 /* Configure for OS presence */
1315 igb_init_manageability(adapter);
1317 /* Prepare transmit descriptors and buffers */
1318 igb_setup_transmit_structures(adapter);
1319 igb_initialize_transmit_units(adapter);
1321 /* Setup Multicast table */
1322 igb_set_multi(adapter);
1325 ** Figure out the desired mbuf pool
1326 ** for doing jumbo/packetsplit
1328 if (adapter->max_frame_size <= 2048)
1329 adapter->rx_mbuf_sz = MCLBYTES;
1330 else if (adapter->max_frame_size <= 4096)
1331 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1333 adapter->rx_mbuf_sz = MJUM9BYTES;
1335 /* Prepare receive descriptors and buffers */
1336 if (igb_setup_receive_structures(adapter)) {
1337 device_printf(dev, "Could not setup receive structures\n");
1340 igb_initialize_receive_units(adapter);
1342 /* Enable VLAN support */
1343 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1344 igb_setup_vlan_hw_support(adapter);
1346 /* Don't lose promiscuous settings */
1347 igb_set_promisc(adapter);
1349 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1350 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1352 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1353 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1355 if (adapter->msix > 1) /* Set up queue routing */
1356 igb_configure_queues(adapter);
1358 /* this clears any pending interrupts */
1359 E1000_READ_REG(&adapter->hw, E1000_ICR);
1360 #ifdef DEVICE_POLLING
1362 * Only enable interrupts if we are not polling, make sure
1363 * they are off otherwise.
1365 if (ifp->if_capenable & IFCAP_POLLING)
1366 igb_disable_intr(adapter);
1368 #endif /* DEVICE_POLLING */
1370 igb_enable_intr(adapter);
1371 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1374 /* Set Energy Efficient Ethernet */
1375 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
1376 if (adapter->hw.mac.type == e1000_i354)
1377 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
1379 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
1386 struct adapter *adapter = arg;
1388 IGB_CORE_LOCK(adapter);
1389 igb_init_locked(adapter);
1390 IGB_CORE_UNLOCK(adapter);
1395 igb_handle_que(void *context, int pending)
1397 struct igb_queue *que = context;
1398 struct adapter *adapter = que->adapter;
1399 struct tx_ring *txr = que->txr;
1400 struct ifnet *ifp = adapter->ifp;
1402 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1405 more = igb_rxeof(que, adapter->rx_process_limit, NULL);
1409 #ifndef IGB_LEGACY_TX
1410 /* Process the stack queue only if not depleted */
1411 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1412 !drbr_empty(ifp, txr->br))
1413 igb_mq_start_locked(ifp, txr);
1415 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1416 igb_start_locked(txr, ifp);
1419 /* Do we need another? */
1421 taskqueue_enqueue(que->tq, &que->que_task);
1426 #ifdef DEVICE_POLLING
1427 if (ifp->if_capenable & IFCAP_POLLING)
1430 /* Reenable this interrupt */
1432 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1434 igb_enable_intr(adapter);
1437 /* Deal with link in a sleepable context */
1439 igb_handle_link(void *context, int pending)
1441 struct adapter *adapter = context;
1443 IGB_CORE_LOCK(adapter);
1444 igb_handle_link_locked(adapter);
1445 IGB_CORE_UNLOCK(adapter);
1449 igb_handle_link_locked(struct adapter *adapter)
1451 struct tx_ring *txr = adapter->tx_rings;
1452 struct ifnet *ifp = adapter->ifp;
1454 IGB_CORE_LOCK_ASSERT(adapter);
1455 adapter->hw.mac.get_link_status = 1;
1456 igb_update_link_status(adapter);
1457 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
1458 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1460 #ifndef IGB_LEGACY_TX
1461 /* Process the stack queue only if not depleted */
1462 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1463 !drbr_empty(ifp, txr->br))
1464 igb_mq_start_locked(ifp, txr);
1466 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1467 igb_start_locked(txr, ifp);
1474 /*********************************************************************
1476 * MSI/Legacy Deferred
1477 * Interrupt Service routine
1479 *********************************************************************/
1481 igb_irq_fast(void *arg)
1483 struct adapter *adapter = arg;
1484 struct igb_queue *que = adapter->queues;
1488 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1491 if (reg_icr == 0xffffffff)
1492 return FILTER_STRAY;
1494 /* Definitely not our interrupt. */
1496 return FILTER_STRAY;
1498 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1499 return FILTER_STRAY;
1502 * Mask interrupts until the taskqueue is finished running. This is
1503 * cheap, just assume that it is needed. This also works around the
1504 * MSI message reordering errata on certain systems.
1506 igb_disable_intr(adapter);
1507 taskqueue_enqueue(que->tq, &que->que_task);
1509 /* Link status change */
1510 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1511 taskqueue_enqueue(que->tq, &adapter->link_task);
1513 if (reg_icr & E1000_ICR_RXO)
1514 adapter->rx_overruns++;
1515 return FILTER_HANDLED;
1518 #ifdef DEVICE_POLLING
1519 #if __FreeBSD_version >= 800000
1520 #define POLL_RETURN_COUNT(a) (a)
1523 #define POLL_RETURN_COUNT(a)
1526 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1528 struct adapter *adapter = ifp->if_softc;
1529 struct igb_queue *que;
1530 struct tx_ring *txr;
1531 u32 reg_icr, rx_done = 0;
1532 u32 loop = IGB_MAX_LOOP;
1535 IGB_CORE_LOCK(adapter);
1536 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1537 IGB_CORE_UNLOCK(adapter);
1538 return POLL_RETURN_COUNT(rx_done);
1541 if (cmd == POLL_AND_CHECK_STATUS) {
1542 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1543 /* Link status change */
1544 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1545 igb_handle_link_locked(adapter);
1547 if (reg_icr & E1000_ICR_RXO)
1548 adapter->rx_overruns++;
1550 IGB_CORE_UNLOCK(adapter);
1552 for (int i = 0; i < adapter->num_queues; i++) {
1553 que = &adapter->queues[i];
1556 igb_rxeof(que, count, &rx_done);
1560 more = igb_txeof(txr);
1561 } while (loop-- && more);
1562 #ifndef IGB_LEGACY_TX
1563 if (!drbr_empty(ifp, txr->br))
1564 igb_mq_start_locked(ifp, txr);
1566 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1567 igb_start_locked(txr, ifp);
1572 return POLL_RETURN_COUNT(rx_done);
1574 #endif /* DEVICE_POLLING */
1576 /*********************************************************************
1578 * MSIX Que Interrupt Service routine
1580 **********************************************************************/
1582 igb_msix_que(void *arg)
1584 struct igb_queue *que = arg;
1585 struct adapter *adapter = que->adapter;
1586 struct ifnet *ifp = adapter->ifp;
1587 struct tx_ring *txr = que->txr;
1588 struct rx_ring *rxr = que->rxr;
1592 /* Ignore spurious interrupts */
1593 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1596 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1601 #ifndef IGB_LEGACY_TX
1602 /* Process the stack queue only if not depleted */
1603 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
1604 !drbr_empty(ifp, txr->br))
1605 igb_mq_start_locked(ifp, txr);
1607 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1608 igb_start_locked(txr, ifp);
1612 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
1614 if (adapter->enable_aim == FALSE)
1617 ** Do Adaptive Interrupt Moderation:
1618 ** - Write out last calculated setting
1619 ** - Calculate based on average size over
1620 ** the last interval.
1622 if (que->eitr_setting)
1623 E1000_WRITE_REG(&adapter->hw,
1624 E1000_EITR(que->msix), que->eitr_setting);
1626 que->eitr_setting = 0;
1628 /* Idle, do nothing */
1629 if ((txr->bytes == 0) && (rxr->bytes == 0))
1632 /* Used half Default if sub-gig */
1633 if (adapter->link_speed != 1000)
1634 newitr = IGB_DEFAULT_ITR / 2;
1636 if ((txr->bytes) && (txr->packets))
1637 newitr = txr->bytes/txr->packets;
1638 if ((rxr->bytes) && (rxr->packets))
1639 newitr = max(newitr,
1640 (rxr->bytes / rxr->packets));
1641 newitr += 24; /* account for hardware frame, crc */
1642 /* set an upper boundary */
1643 newitr = min(newitr, 3000);
1644 /* Be nice to the mid range */
1645 if ((newitr > 300) && (newitr < 1200))
1646 newitr = (newitr / 3);
1648 newitr = (newitr / 2);
1650 newitr &= 0x7FFC; /* Mask invalid bits */
1651 if (adapter->hw.mac.type == e1000_82575)
1652 newitr |= newitr << 16;
1654 newitr |= E1000_EITR_CNT_IGNR;
1656 /* save for next interrupt */
1657 que->eitr_setting = newitr;
1666 /* Schedule a clean task if needed*/
1668 taskqueue_enqueue(que->tq, &que->que_task);
1670 /* Reenable this interrupt */
1671 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1676 /*********************************************************************
1678 * MSIX Link Interrupt Service routine
1680 **********************************************************************/
1683 igb_msix_link(void *arg)
1685 struct adapter *adapter = arg;
1688 ++adapter->link_irq;
1689 icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1690 if (!(icr & E1000_ICR_LSC))
1692 igb_handle_link(adapter, 0);
1696 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1697 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1702 /*********************************************************************
1704 * Media Ioctl callback
1706 * This routine is called whenever the user queries the status of
1707 * the interface using ifconfig.
1709 **********************************************************************/
1711 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1713 struct adapter *adapter = ifp->if_softc;
1715 INIT_DEBUGOUT("igb_media_status: begin");
1717 IGB_CORE_LOCK(adapter);
1718 igb_update_link_status(adapter);
1720 ifmr->ifm_status = IFM_AVALID;
1721 ifmr->ifm_active = IFM_ETHER;
1723 if (!adapter->link_active) {
1724 IGB_CORE_UNLOCK(adapter);
1728 ifmr->ifm_status |= IFM_ACTIVE;
1730 switch (adapter->link_speed) {
1732 ifmr->ifm_active |= IFM_10_T;
1736 ** Support for 100Mb SFP - these are Fiber
1737 ** but the media type appears as serdes
1739 if (adapter->hw.phy.media_type ==
1740 e1000_media_type_internal_serdes)
1741 ifmr->ifm_active |= IFM_100_FX;
1743 ifmr->ifm_active |= IFM_100_TX;
1746 ifmr->ifm_active |= IFM_1000_T;
1749 ifmr->ifm_active |= IFM_2500_SX;
1753 if (adapter->link_duplex == FULL_DUPLEX)
1754 ifmr->ifm_active |= IFM_FDX;
1756 ifmr->ifm_active |= IFM_HDX;
1758 IGB_CORE_UNLOCK(adapter);
1761 /*********************************************************************
1763 * Media Ioctl callback
1765 * This routine is called when the user changes speed/duplex using
1766 * media/mediopt option with ifconfig.
1768 **********************************************************************/
1770 igb_media_change(struct ifnet *ifp)
1772 struct adapter *adapter = ifp->if_softc;
1773 struct ifmedia *ifm = &adapter->media;
1775 INIT_DEBUGOUT("igb_media_change: begin");
1777 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1780 IGB_CORE_LOCK(adapter);
1781 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1783 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1784 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1789 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1790 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1793 adapter->hw.mac.autoneg = FALSE;
1794 adapter->hw.phy.autoneg_advertised = 0;
1795 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1796 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1798 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1801 adapter->hw.mac.autoneg = FALSE;
1802 adapter->hw.phy.autoneg_advertised = 0;
1803 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1804 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1806 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1809 device_printf(adapter->dev, "Unsupported media type\n");
1812 igb_init_locked(adapter);
1813 IGB_CORE_UNLOCK(adapter);
1819 /*********************************************************************
1821 * This routine maps the mbufs to Advanced TX descriptors.
1823 **********************************************************************/
1825 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1827 struct adapter *adapter = txr->adapter;
1828 u32 olinfo_status = 0, cmd_type_len;
1829 int i, j, error, nsegs;
1832 struct mbuf *m_head;
1833 bus_dma_segment_t segs[IGB_MAX_SCATTER];
1835 struct igb_tx_buf *txbuf;
1836 union e1000_adv_tx_desc *txd = NULL;
1840 /* Basic descriptor defines */
1841 cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
1842 E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
1844 if (m_head->m_flags & M_VLANTAG)
1845 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1848 * Important to capture the first descriptor
1849 * used because it will contain the index of
1850 * the one we tell the hardware to report back
1852 first = txr->next_avail_desc;
1853 txbuf = &txr->tx_buffers[first];
1857 * Map the packet for DMA.
1860 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1861 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1863 if (__predict_false(error)) {
1868 /* Try it again? - one try */
1869 if (remap == TRUE) {
1871 m = m_collapse(*m_headp, M_NOWAIT,
1874 adapter->mbuf_defrag_failed++;
1884 txr->no_tx_dma_setup++;
1891 /* Make certain there are enough descriptors */
1892 if (txr->tx_avail < (nsegs + 2)) {
1893 txr->no_desc_avail++;
1894 bus_dmamap_unload(txr->txtag, map);
1900 ** Set up the appropriate offload context
1901 ** this will consume the first descriptor
1903 error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1904 if (__predict_false(error)) {
1910 /* 82575 needs the queue index added */
1911 if (adapter->hw.mac.type == e1000_82575)
1912 olinfo_status |= txr->me << 4;
1914 i = txr->next_avail_desc;
1915 for (j = 0; j < nsegs; j++) {
1919 txbuf = &txr->tx_buffers[i];
1920 txd = &txr->tx_base[i];
1921 seglen = segs[j].ds_len;
1922 segaddr = htole64(segs[j].ds_addr);
1924 txd->read.buffer_addr = segaddr;
1925 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
1926 cmd_type_len | seglen);
1927 txd->read.olinfo_status = htole32(olinfo_status);
1929 if (++i == txr->num_desc)
1933 txd->read.cmd_type_len |=
1934 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1935 txr->tx_avail -= nsegs;
1936 txr->next_avail_desc = i;
1938 txbuf->m_head = m_head;
1940 ** Here we swap the map so the last descriptor,
1941 ** which gets the completion interrupt has the
1942 ** real map, and the first descriptor gets the
1943 ** unused map from this descriptor.
1945 txr->tx_buffers[first].map = txbuf->map;
1947 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1949 /* Set the EOP descriptor that will be marked done */
1950 txbuf = &txr->tx_buffers[first];
1953 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1954 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1956 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1957 * hardware that this frame is available to transmit.
1959 ++txr->total_packets;
1960 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1965 igb_set_promisc(struct adapter *adapter)
1967 struct ifnet *ifp = adapter->ifp;
1968 struct e1000_hw *hw = &adapter->hw;
1971 if (adapter->vf_ifp) {
1972 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
1976 reg = E1000_READ_REG(hw, E1000_RCTL);
1977 if (ifp->if_flags & IFF_PROMISC) {
1978 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1979 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1980 } else if (ifp->if_flags & IFF_ALLMULTI) {
1981 reg |= E1000_RCTL_MPE;
1982 reg &= ~E1000_RCTL_UPE;
1983 E1000_WRITE_REG(hw, E1000_RCTL, reg);
1988 igb_disable_promisc(struct adapter *adapter)
1990 struct e1000_hw *hw = &adapter->hw;
1991 struct ifnet *ifp = adapter->ifp;
1995 if (adapter->vf_ifp) {
1996 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
1999 reg = E1000_READ_REG(hw, E1000_RCTL);
2000 reg &= (~E1000_RCTL_UPE);
2001 if (ifp->if_flags & IFF_ALLMULTI)
2002 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2004 struct ifmultiaddr *ifma;
2005 #if __FreeBSD_version < 800000
2008 if_maddr_rlock(ifp);
2010 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2011 if (ifma->ifma_addr->sa_family != AF_LINK)
2013 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2017 #if __FreeBSD_version < 800000
2018 IF_ADDR_UNLOCK(ifp);
2020 if_maddr_runlock(ifp);
2023 /* Don't disable if in MAX groups */
2024 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2025 reg &= (~E1000_RCTL_MPE);
2026 E1000_WRITE_REG(hw, E1000_RCTL, reg);
2030 /*********************************************************************
2033 * This routine is called whenever multicast address list is updated.
2035 **********************************************************************/
2038 igb_set_multi(struct adapter *adapter)
2040 struct ifnet *ifp = adapter->ifp;
2041 struct ifmultiaddr *ifma;
2047 IOCTL_DEBUGOUT("igb_set_multi: begin");
2050 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
2051 MAX_NUM_MULTICAST_ADDRESSES);
2053 #if __FreeBSD_version < 800000
2056 if_maddr_rlock(ifp);
2058 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2059 if (ifma->ifma_addr->sa_family != AF_LINK)
2062 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2065 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2066 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2069 #if __FreeBSD_version < 800000
2070 IF_ADDR_UNLOCK(ifp);
2072 if_maddr_runlock(ifp);
2075 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2076 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2077 reg_rctl |= E1000_RCTL_MPE;
2078 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2080 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2084 /*********************************************************************
2086 * This routine checks for link status,
2087 * updates statistics, and does the watchdog.
2089 **********************************************************************/
2092 igb_local_timer(void *arg)
2094 struct adapter *adapter = arg;
2095 device_t dev = adapter->dev;
2096 struct ifnet *ifp = adapter->ifp;
2097 struct tx_ring *txr = adapter->tx_rings;
2098 struct igb_queue *que = adapter->queues;
2099 int hung = 0, busy = 0;
2102 IGB_CORE_LOCK_ASSERT(adapter);
2104 igb_update_link_status(adapter);
2105 igb_update_stats_counters(adapter);
2108 ** Check the TX queues status
2109 ** - central locked handling of OACTIVE
2110 ** - watchdog only if all queues show hung
2112 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2113 if ((txr->queue_status & IGB_QUEUE_HUNG) &&
2114 (adapter->pause_frames == 0))
2116 if (txr->queue_status & IGB_QUEUE_DEPLETED)
2118 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
2119 taskqueue_enqueue(que->tq, &que->que_task);
2121 if (hung == adapter->num_queues)
2123 if (busy == adapter->num_queues)
2124 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2125 else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
2126 (busy < adapter->num_queues))
2127 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2129 adapter->pause_frames = 0;
2130 callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
2131 #ifndef DEVICE_POLLING
2132 /* Schedule all queue interrupts - deadlock protection */
2133 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
2138 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2139 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2140 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2141 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2142 device_printf(dev,"TX(%d) desc avail = %d,"
2143 "Next TX to Clean = %d\n",
2144 txr->me, txr->tx_avail, txr->next_to_clean);
2145 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2146 adapter->watchdog_events++;
2147 igb_init_locked(adapter);
2151 igb_update_link_status(struct adapter *adapter)
2153 struct e1000_hw *hw = &adapter->hw;
2154 struct e1000_fc_info *fc = &hw->fc;
2155 struct ifnet *ifp = adapter->ifp;
2156 device_t dev = adapter->dev;
2157 struct tx_ring *txr = adapter->tx_rings;
2158 u32 link_check, thstat, ctrl;
2159 char *flowctl = NULL;
2161 link_check = thstat = ctrl = 0;
2163 /* Get the cached link value or read for real */
2164 switch (hw->phy.media_type) {
2165 case e1000_media_type_copper:
2166 if (hw->mac.get_link_status) {
2167 /* Do the work to read phy */
2168 e1000_check_for_link(hw);
2169 link_check = !hw->mac.get_link_status;
2173 case e1000_media_type_fiber:
2174 e1000_check_for_link(hw);
2175 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2178 case e1000_media_type_internal_serdes:
2179 e1000_check_for_link(hw);
2180 link_check = adapter->hw.mac.serdes_has_link;
2182 /* VF device is type_unknown */
2183 case e1000_media_type_unknown:
2184 e1000_check_for_link(hw);
2185 link_check = !hw->mac.get_link_status;
2191 /* Check for thermal downshift or shutdown */
2192 if (hw->mac.type == e1000_i350) {
2193 thstat = E1000_READ_REG(hw, E1000_THSTAT);
2194 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
2197 /* Get the flow control for display */
2198 switch (fc->current_mode) {
2199 case e1000_fc_rx_pause:
2202 case e1000_fc_tx_pause:
2214 /* Now we check if a transition has happened */
2215 if (link_check && (adapter->link_active == 0)) {
2216 e1000_get_speed_and_duplex(&adapter->hw,
2217 &adapter->link_speed, &adapter->link_duplex);
2219 device_printf(dev, "Link is up %d Mbps %s,"
2220 " Flow Control: %s\n",
2221 adapter->link_speed,
2222 ((adapter->link_duplex == FULL_DUPLEX) ?
2223 "Full Duplex" : "Half Duplex"), flowctl);
2224 adapter->link_active = 1;
2225 ifp->if_baudrate = adapter->link_speed * 1000000;
2226 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2227 (thstat & E1000_THSTAT_LINK_THROTTLE))
2228 device_printf(dev, "Link: thermal downshift\n");
2229 /* Delay Link Up for Phy update */
2230 if (((hw->mac.type == e1000_i210) ||
2231 (hw->mac.type == e1000_i211)) &&
2232 (hw->phy.id == I210_I_PHY_ID))
2233 msec_delay(I210_LINK_DELAY);
2234 /* Reset if the media type changed. */
2235 if (hw->dev_spec._82575.media_changed) {
2236 hw->dev_spec._82575.media_changed = false;
2237 adapter->flags |= IGB_MEDIA_RESET;
2240 /* This can sleep */
2241 if_link_state_change(ifp, LINK_STATE_UP);
2242 } else if (!link_check && (adapter->link_active == 1)) {
2243 ifp->if_baudrate = adapter->link_speed = 0;
2244 adapter->link_duplex = 0;
2246 device_printf(dev, "Link is Down\n");
2247 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
2248 (thstat & E1000_THSTAT_PWR_DOWN))
2249 device_printf(dev, "Link: thermal shutdown\n");
2250 adapter->link_active = 0;
2251 /* This can sleep */
2252 if_link_state_change(ifp, LINK_STATE_DOWN);
2253 /* Reset queue state */
2254 for (int i = 0; i < adapter->num_queues; i++, txr++)
2255 txr->queue_status = IGB_QUEUE_IDLE;
2259 /*********************************************************************
2261 * This routine disables all traffic on the adapter by issuing a
2262 * global reset on the MAC and deallocates TX/RX buffers.
2264 **********************************************************************/
2269 struct adapter *adapter = arg;
2270 struct ifnet *ifp = adapter->ifp;
2271 struct tx_ring *txr = adapter->tx_rings;
2273 IGB_CORE_LOCK_ASSERT(adapter);
2275 INIT_DEBUGOUT("igb_stop: begin");
2277 igb_disable_intr(adapter);
2279 callout_stop(&adapter->timer);
2281 /* Tell the stack that the interface is no longer active */
2282 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2283 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2285 /* Disarm watchdog timer. */
2286 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2288 txr->queue_status = IGB_QUEUE_IDLE;
2292 e1000_reset_hw(&adapter->hw);
2293 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2295 e1000_led_off(&adapter->hw);
2296 e1000_cleanup_led(&adapter->hw);
2300 /*********************************************************************
2302 * Determine hardware revision.
2304 **********************************************************************/
2306 igb_identify_hardware(struct adapter *adapter)
2308 device_t dev = adapter->dev;
2310 /* Make sure our PCI config space has the necessary stuff set */
2311 pci_enable_busmaster(dev);
2312 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2314 /* Save off the information about this board */
2315 adapter->hw.vendor_id = pci_get_vendor(dev);
2316 adapter->hw.device_id = pci_get_device(dev);
2317 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2318 adapter->hw.subsystem_vendor_id =
2319 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2320 adapter->hw.subsystem_device_id =
2321 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2323 /* Set MAC type early for PCI setup */
2324 e1000_set_mac_type(&adapter->hw);
2326 /* Are we a VF device? */
2327 if ((adapter->hw.mac.type == e1000_vfadapt) ||
2328 (adapter->hw.mac.type == e1000_vfadapt_i350))
2329 adapter->vf_ifp = 1;
2331 adapter->vf_ifp = 0;
2335 igb_allocate_pci_resources(struct adapter *adapter)
2337 device_t dev = adapter->dev;
2341 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2343 if (adapter->pci_mem == NULL) {
2344 device_printf(dev, "Unable to allocate bus resource: memory\n");
2347 adapter->osdep.mem_bus_space_tag =
2348 rman_get_bustag(adapter->pci_mem);
2349 adapter->osdep.mem_bus_space_handle =
2350 rman_get_bushandle(adapter->pci_mem);
2351 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2353 adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2355 /* This will setup either MSI/X or MSI */
2356 adapter->msix = igb_setup_msix(adapter);
2357 adapter->hw.back = &adapter->osdep;
2362 /*********************************************************************
2364 * Setup the Legacy or MSI Interrupt handler
2366 **********************************************************************/
2368 igb_allocate_legacy(struct adapter *adapter)
2370 device_t dev = adapter->dev;
2371 struct igb_queue *que = adapter->queues;
2372 #ifndef IGB_LEGACY_TX
2373 struct tx_ring *txr = adapter->tx_rings;
2377 /* Turn off all interrupts */
2378 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2381 if (adapter->msix == 1)
2384 /* We allocate a single interrupt resource */
2385 adapter->res = bus_alloc_resource_any(dev,
2386 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2387 if (adapter->res == NULL) {
2388 device_printf(dev, "Unable to allocate bus resource: "
2393 #ifndef IGB_LEGACY_TX
2394 TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
2398 * Try allocating a fast interrupt and the associated deferred
2399 * processing contexts.
2401 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2402 /* Make tasklet for deferred link handling */
2403 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2404 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2405 taskqueue_thread_enqueue, &que->tq);
2406 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
2407 device_get_nameunit(adapter->dev));
2408 if ((error = bus_setup_intr(dev, adapter->res,
2409 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2410 adapter, &adapter->tag)) != 0) {
2411 device_printf(dev, "Failed to register fast interrupt "
2412 "handler: %d\n", error);
2413 taskqueue_free(que->tq);
2422 /*********************************************************************
2424 * Setup the MSIX Queue Interrupt handlers:
2426 **********************************************************************/
2428 igb_allocate_msix(struct adapter *adapter)
2430 device_t dev = adapter->dev;
2431 struct igb_queue *que = adapter->queues;
2432 int error, rid, vector = 0;
2438 /* Be sure to start with all interrupts disabled */
2439 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
2440 E1000_WRITE_FLUSH(&adapter->hw);
2444 * If we're doing RSS, the number of queues needs to
2445 * match the number of RSS buckets that are configured.
2447 * + If there's more queues than RSS buckets, we'll end
2448 * up with queues that get no traffic.
2450 * + If there's more RSS buckets than queues, we'll end
2451 * up having multiple RSS buckets map to the same queue,
2452 * so there'll be some contention.
2454 if (adapter->num_queues != rss_getnumbuckets()) {
2456 "%s: number of queues (%d) != number of RSS buckets (%d)"
2457 "; performance will be impacted.\n",
2459 adapter->num_queues,
2460 rss_getnumbuckets());
2464 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2466 que->res = bus_alloc_resource_any(dev,
2467 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2468 if (que->res == NULL) {
2470 "Unable to allocate bus resource: "
2471 "MSIX Queue Interrupt\n");
2474 error = bus_setup_intr(dev, que->res,
2475 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2476 igb_msix_que, que, &que->tag);
2479 device_printf(dev, "Failed to register Queue handler");
2482 #if __FreeBSD_version >= 800504
2483 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2486 if (adapter->hw.mac.type == e1000_82575)
2487 que->eims = E1000_EICR_TX_QUEUE0 << i;
2489 que->eims = 1 << vector;
2493 * The queue ID is used as the RSS layer bucket ID.
2494 * We look up the queue ID -> RSS CPU ID and select
2497 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2500 * Bind the msix vector, and thus the
2501 * rings to the corresponding cpu.
2503 * This just happens to match the default RSS round-robin
2504 * bucket -> queue -> CPU allocation.
2506 if (adapter->num_queues > 1) {
2507 if (igb_last_bind_cpu < 0)
2508 igb_last_bind_cpu = CPU_FIRST();
2509 cpu_id = igb_last_bind_cpu;
2513 if (adapter->num_queues > 1) {
2514 bus_bind_intr(dev, que->res, cpu_id);
2517 "Bound queue %d to RSS bucket %d\n",
2521 "Bound queue %d to cpu %d\n",
2526 #ifndef IGB_LEGACY_TX
2527 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
2530 /* Make tasklet for deferred handling */
2531 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2532 que->tq = taskqueue_create("igb_que", M_NOWAIT,
2533 taskqueue_thread_enqueue, &que->tq);
2534 if (adapter->num_queues > 1) {
2536 * Only pin the taskqueue thread to a CPU if
2539 * This again just happens to match the default RSS
2540 * round-robin bucket -> queue -> CPU allocation.
2543 CPU_SETOF(cpu_id, &cpu_mask);
2544 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2546 "%s que (bucket %d)",
2547 device_get_nameunit(adapter->dev),
2550 taskqueue_start_threads(&que->tq, 1, PI_NET,
2552 device_get_nameunit(adapter->dev),
2556 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2557 device_get_nameunit(adapter->dev));
2560 /* Finally update the last bound CPU id */
2561 if (adapter->num_queues > 1)
2562 igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
2567 adapter->res = bus_alloc_resource_any(dev,
2568 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2569 if (adapter->res == NULL) {
2571 "Unable to allocate bus resource: "
2572 "MSIX Link Interrupt\n");
2575 if ((error = bus_setup_intr(dev, adapter->res,
2576 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2577 igb_msix_link, adapter, &adapter->tag)) != 0) {
2578 device_printf(dev, "Failed to register Link handler");
2581 #if __FreeBSD_version >= 800504
2582 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2584 adapter->linkvec = vector;
2591 igb_configure_queues(struct adapter *adapter)
2593 struct e1000_hw *hw = &adapter->hw;
2594 struct igb_queue *que;
2595 u32 tmp, ivar = 0, newitr = 0;
2597 /* First turn on RSS capability */
2598 if (adapter->hw.mac.type != e1000_82575)
2599 E1000_WRITE_REG(hw, E1000_GPIE,
2600 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2601 E1000_GPIE_PBA | E1000_GPIE_NSICR);
2604 switch (adapter->hw.mac.type) {
2611 case e1000_vfadapt_i350:
2613 for (int i = 0; i < adapter->num_queues; i++) {
2615 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2616 que = &adapter->queues[i];
2619 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2622 ivar |= que->msix | E1000_IVAR_VALID;
2624 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2627 for (int i = 0; i < adapter->num_queues; i++) {
2629 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2630 que = &adapter->queues[i];
2633 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2636 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2638 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2639 adapter->que_mask |= que->eims;
2642 /* And for the link interrupt */
2643 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2644 adapter->link_mask = 1 << adapter->linkvec;
2645 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2649 for (int i = 0; i < adapter->num_queues; i++) {
2650 u32 index = i & 0x7; /* Each IVAR has two entries */
2651 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2652 que = &adapter->queues[i];
2655 ivar |= que->msix | E1000_IVAR_VALID;
2658 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2660 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2661 adapter->que_mask |= que->eims;
2664 for (int i = 0; i < adapter->num_queues; i++) {
2665 u32 index = i & 0x7; /* Each IVAR has two entries */
2666 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2667 que = &adapter->queues[i];
2670 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2673 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2675 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2676 adapter->que_mask |= que->eims;
2679 /* And for the link interrupt */
2680 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2681 adapter->link_mask = 1 << adapter->linkvec;
2682 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2686 /* enable MSI-X support*/
2687 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2688 tmp |= E1000_CTRL_EXT_PBA_CLR;
2689 /* Auto-Mask interrupts upon ICR read. */
2690 tmp |= E1000_CTRL_EXT_EIAME;
2691 tmp |= E1000_CTRL_EXT_IRCA;
2692 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2695 for (int i = 0; i < adapter->num_queues; i++) {
2696 que = &adapter->queues[i];
2697 tmp = E1000_EICR_RX_QUEUE0 << i;
2698 tmp |= E1000_EICR_TX_QUEUE0 << i;
2700 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2702 adapter->que_mask |= que->eims;
2706 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2708 adapter->link_mask |= E1000_EIMS_OTHER;
2713 /* Set the starting interrupt rate */
2714 if (igb_max_interrupt_rate > 0)
2715 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
2717 if (hw->mac.type == e1000_82575)
2718 newitr |= newitr << 16;
2720 newitr |= E1000_EITR_CNT_IGNR;
2722 for (int i = 0; i < adapter->num_queues; i++) {
2723 que = &adapter->queues[i];
2724 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2732 igb_free_pci_resources(struct adapter *adapter)
2734 struct igb_queue *que = adapter->queues;
2735 device_t dev = adapter->dev;
2739 ** There is a slight possibility of a failure mode
2740 ** in attach that will result in entering this function
2741 ** before interrupt resources have been initialized, and
2742 ** in that case we do not want to execute the loops below
2743 ** We can detect this reliably by the state of the adapter
2746 if (adapter->res == NULL)
2750 * First release all the interrupt resources:
2752 for (int i = 0; i < adapter->num_queues; i++, que++) {
2753 rid = que->msix + 1;
2754 if (que->tag != NULL) {
2755 bus_teardown_intr(dev, que->res, que->tag);
2758 if (que->res != NULL)
2759 bus_release_resource(dev,
2760 SYS_RES_IRQ, rid, que->res);
2763 /* Clean the Legacy or Link interrupt last */
2764 if (adapter->linkvec) /* we are doing MSIX */
2765 rid = adapter->linkvec + 1;
2767 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2769 que = adapter->queues;
2770 if (adapter->tag != NULL) {
2771 taskqueue_drain(que->tq, &adapter->link_task);
2772 bus_teardown_intr(dev, adapter->res, adapter->tag);
2773 adapter->tag = NULL;
2775 if (adapter->res != NULL)
2776 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2778 for (int i = 0; i < adapter->num_queues; i++, que++) {
2779 if (que->tq != NULL) {
2780 #ifndef IGB_LEGACY_TX
2781 taskqueue_drain(que->tq, &que->txr->txq_task);
2783 taskqueue_drain(que->tq, &que->que_task);
2784 taskqueue_free(que->tq);
2789 pci_release_msi(dev);
2791 if (adapter->msix_mem != NULL)
2792 bus_release_resource(dev, SYS_RES_MEMORY,
2793 adapter->memrid, adapter->msix_mem);
2795 if (adapter->pci_mem != NULL)
2796 bus_release_resource(dev, SYS_RES_MEMORY,
2797 PCIR_BAR(0), adapter->pci_mem);
2802 * Setup Either MSI/X or MSI
2805 igb_setup_msix(struct adapter *adapter)
2807 device_t dev = adapter->dev;
2808 int bar, want, queues, msgs, maxqueues;
2810 /* tuneable override */
2811 if (igb_enable_msix == 0)
2814 /* First try MSI/X */
2815 msgs = pci_msix_count(dev);
2819 ** Some new devices, as with ixgbe, now may
2820 ** use a different BAR, so we need to keep
2821 ** track of which is used.
2823 adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
2824 bar = pci_read_config(dev, adapter->memrid, 4);
2825 if (bar == 0) /* use next bar */
2826 adapter->memrid += 4;
2827 adapter->msix_mem = bus_alloc_resource_any(dev,
2828 SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
2829 if (adapter->msix_mem == NULL) {
2830 /* May not be enabled */
2831 device_printf(adapter->dev,
2832 "Unable to map MSIX table \n");
2836 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2838 /* Override via tuneable */
2839 if (igb_num_queues != 0)
2840 queues = igb_num_queues;
2843 /* If we're doing RSS, clamp at the number of RSS buckets */
2844 if (queues > rss_getnumbuckets())
2845 queues = rss_getnumbuckets();
2849 /* Sanity check based on HW */
2850 switch (adapter->hw.mac.type) {
2866 default: /* VF interfaces */
2871 /* Final clamp on the actual hardware capability */
2872 if (queues > maxqueues)
2876 ** One vector (RX/TX pair) per queue
2877 ** plus an additional for Link interrupt
2883 device_printf(adapter->dev,
2884 "MSIX Configuration Problem, "
2885 "%d vectors configured, but %d queues wanted!\n",
2889 if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2890 device_printf(adapter->dev,
2891 "Using MSIX interrupts with %d vectors\n", msgs);
2892 adapter->num_queues = queues;
2896 ** If MSIX alloc failed or provided us with
2897 ** less than needed, free and fall through to MSI
2899 pci_release_msi(dev);
2902 if (adapter->msix_mem != NULL) {
2903 bus_release_resource(dev, SYS_RES_MEMORY,
2904 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2905 adapter->msix_mem = NULL;
2908 if (pci_alloc_msi(dev, &msgs) == 0) {
2909 device_printf(adapter->dev," Using an MSI interrupt\n");
2912 device_printf(adapter->dev," Using a Legacy interrupt\n");
2916 /*********************************************************************
2918 * Initialize the DMA Coalescing feature
2920 **********************************************************************/
2922 igb_init_dmac(struct adapter *adapter, u32 pba)
2924 device_t dev = adapter->dev;
2925 struct e1000_hw *hw = &adapter->hw;
2926 u32 dmac, reg = ~E1000_DMACR_DMAC_EN;
2929 if (hw->mac.type == e1000_i211)
2932 if (hw->mac.type > e1000_82580) {
2934 if (adapter->dmac == 0) { /* Disabling it */
2935 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2938 device_printf(dev, "DMA Coalescing enabled\n");
2940 /* Set starting threshold */
2941 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
2943 hwm = 64 * pba - adapter->max_frame_size / 16;
2944 if (hwm < 64 * (pba - 6))
2945 hwm = 64 * (pba - 6);
2946 reg = E1000_READ_REG(hw, E1000_FCRTC);
2947 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
2948 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
2949 & E1000_FCRTC_RTH_COAL_MASK);
2950 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
2953 dmac = pba - adapter->max_frame_size / 512;
2954 if (dmac < pba - 10)
2956 reg = E1000_READ_REG(hw, E1000_DMACR);
2957 reg &= ~E1000_DMACR_DMACTHR_MASK;
2958 reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
2959 & E1000_DMACR_DMACTHR_MASK);
2961 /* transition to L0x or L1 if available..*/
2962 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
2964 /* Check if status is 2.5Gb backplane connection
2965 * before configuration of watchdog timer, which is
2966 * in msec values in 12.8usec intervals
2967 * watchdog timer= msec values in 32usec intervals
2968 * for non 2.5Gb connection
2970 if (hw->mac.type == e1000_i354) {
2971 int status = E1000_READ_REG(hw, E1000_STATUS);
2972 if ((status & E1000_STATUS_2P5_SKU) &&
2973 (!(status & E1000_STATUS_2P5_SKU_OVER)))
2974 reg |= ((adapter->dmac * 5) >> 6);
2976 reg |= (adapter->dmac >> 5);
2978 reg |= (adapter->dmac >> 5);
2981 E1000_WRITE_REG(hw, E1000_DMACR, reg);
2983 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
2985 /* Set the interval before transition */
2986 reg = E1000_READ_REG(hw, E1000_DMCTLX);
2987 if (hw->mac.type == e1000_i350)
2988 reg |= IGB_DMCTLX_DCFLUSH_DIS;
2990 ** in 2.5Gb connection, TTLX unit is 0.4 usec
2991 ** which is 0x4*2 = 0xA. But delay is still 4 usec
2993 if (hw->mac.type == e1000_i354) {
2994 int status = E1000_READ_REG(hw, E1000_STATUS);
2995 if ((status & E1000_STATUS_2P5_SKU) &&
2996 (!(status & E1000_STATUS_2P5_SKU_OVER)))
3004 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
3006 /* free space in tx packet buffer to wake from DMA coal */
3007 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
3008 (2 * adapter->max_frame_size)) >> 6);
3010 /* make low power state decision controlled by DMA coal */
3011 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
3012 reg &= ~E1000_PCIEMISC_LX_DECISION;
3013 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
3015 } else if (hw->mac.type == e1000_82580) {
3016 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
3017 E1000_WRITE_REG(hw, E1000_PCIEMISC,
3018 reg & ~E1000_PCIEMISC_LX_DECISION);
3019 E1000_WRITE_REG(hw, E1000_DMACR, 0);
3024 /*********************************************************************
3026 * Set up an fresh starting state
3028 **********************************************************************/
3030 igb_reset(struct adapter *adapter)
3032 device_t dev = adapter->dev;
3033 struct e1000_hw *hw = &adapter->hw;
3034 struct e1000_fc_info *fc = &hw->fc;
3035 struct ifnet *ifp = adapter->ifp;
3039 INIT_DEBUGOUT("igb_reset: begin");
3041 /* Let the firmware know the OS is in control */
3042 igb_get_hw_control(adapter);
3045 * Packet Buffer Allocation (PBA)
3046 * Writing PBA sets the receive portion of the buffer
3047 * the remainder is used for the transmit buffer.
3049 switch (hw->mac.type) {
3051 pba = E1000_PBA_32K;
3055 pba = E1000_READ_REG(hw, E1000_RXPBS);
3056 pba &= E1000_RXPBS_SIZE_MASK_82576;
3061 case e1000_vfadapt_i350:
3062 pba = E1000_READ_REG(hw, E1000_RXPBS);
3063 pba = e1000_rxpbs_adjust_82580(pba);
3067 pba = E1000_PBA_34K;
3072 /* Special needs in case of Jumbo frames */
3073 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
3074 u32 tx_space, min_tx, min_rx;
3075 pba = E1000_READ_REG(hw, E1000_PBA);
3076 tx_space = pba >> 16;
3078 min_tx = (adapter->max_frame_size +
3079 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
3080 min_tx = roundup2(min_tx, 1024);
3082 min_rx = adapter->max_frame_size;
3083 min_rx = roundup2(min_rx, 1024);
3085 if (tx_space < min_tx &&
3086 ((min_tx - tx_space) < pba)) {
3087 pba = pba - (min_tx - tx_space);
3089 * if short on rx space, rx wins
3090 * and must trump tx adjustment
3095 E1000_WRITE_REG(hw, E1000_PBA, pba);
3098 INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
3101 * These parameters control the automatic generation (Tx) and
3102 * response (Rx) to Ethernet PAUSE frames.
3103 * - High water mark should allow for at least two frames to be
3104 * received after sending an XOFF.
3105 * - Low water mark works best when it is very near the high water mark.
3106 * This allows the receiver to restart by sending XON when it has
3109 hwm = min(((pba << 10) * 9 / 10),
3110 ((pba << 10) - 2 * adapter->max_frame_size));
3112 if (hw->mac.type < e1000_82576) {
3113 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
3114 fc->low_water = fc->high_water - 8;
3116 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
3117 fc->low_water = fc->high_water - 16;
3120 fc->pause_time = IGB_FC_PAUSE_TIME;
3121 fc->send_xon = TRUE;
3123 fc->requested_mode = adapter->fc;
3125 fc->requested_mode = e1000_fc_default;
3127 /* Issue a global reset */
3129 E1000_WRITE_REG(hw, E1000_WUC, 0);
3131 /* Reset for AutoMediaDetect */
3132 if (adapter->flags & IGB_MEDIA_RESET) {
3133 e1000_setup_init_funcs(hw, TRUE);
3134 e1000_get_bus_info(hw);
3135 adapter->flags &= ~IGB_MEDIA_RESET;
3138 if (e1000_init_hw(hw) < 0)
3139 device_printf(dev, "Hardware Initialization Failed\n");
3141 /* Setup DMA Coalescing */
3142 igb_init_dmac(adapter, pba);
3144 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
3145 e1000_get_phy_info(hw);
3146 e1000_check_for_link(hw);
3150 /*********************************************************************
3152 * Setup networking device structure and register an interface.
3154 **********************************************************************/
3156 igb_setup_interface(device_t dev, struct adapter *adapter)
3160 INIT_DEBUGOUT("igb_setup_interface: begin");
3162 ifp = adapter->ifp = if_alloc(IFT_ETHER);
3164 device_printf(dev, "can not allocate ifnet structure\n");
3167 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3168 ifp->if_init = igb_init;
3169 ifp->if_softc = adapter;
3170 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3171 ifp->if_ioctl = igb_ioctl;
3172 ifp->if_get_counter = igb_get_counter;
3174 /* TSO parameters */
3175 ifp->if_hw_tsomax = IP_MAXPACKET;
3176 ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER;
3177 ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE;
3179 #ifndef IGB_LEGACY_TX
3180 ifp->if_transmit = igb_mq_start;
3181 ifp->if_qflush = igb_qflush;
3183 ifp->if_start = igb_start;
3184 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3185 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3186 IFQ_SET_READY(&ifp->if_snd);
3189 ether_ifattach(ifp, adapter->hw.mac.addr);
3191 ifp->if_capabilities = ifp->if_capenable = 0;
3193 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3194 #if __FreeBSD_version >= 1000000
3195 ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
3197 ifp->if_capabilities |= IFCAP_TSO;
3198 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
3199 ifp->if_capenable = ifp->if_capabilities;
3201 /* Don't enable LRO by default */
3202 ifp->if_capabilities |= IFCAP_LRO;
3204 #ifdef DEVICE_POLLING
3205 ifp->if_capabilities |= IFCAP_POLLING;
3209 * Tell the upper layer(s) we
3210 * support full VLAN capability.
3212 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
3213 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
3216 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
3221 ** Don't turn this on by default, if vlans are
3222 ** created on another pseudo device (eg. lagg)
3223 ** then vlan events are not passed thru, breaking
3224 ** operation, but with HW FILTER off it works. If
3225 ** using vlans directly on the igb driver you can
3226 ** enable this and get full hardware tag filtering.
3228 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3231 * Specify the media types supported by this adapter and register
3232 * callbacks to update media and link information
3234 ifmedia_init(&adapter->media, IFM_IMASK,
3235 igb_media_change, igb_media_status);
3236 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3237 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3238 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
3240 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
3242 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3243 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3245 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3247 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3249 if (adapter->hw.phy.type != e1000_phy_ife) {
3250 ifmedia_add(&adapter->media,
3251 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3252 ifmedia_add(&adapter->media,
3253 IFM_ETHER | IFM_1000_T, 0, NULL);
3256 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3257 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3263 * Manage DMA'able memory.
3266 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3270 *(bus_addr_t *) arg = segs[0].ds_addr;
3274 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
3275 struct igb_dma_alloc *dma, int mapflags)
3279 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3280 IGB_DBA_ALIGN, 0, /* alignment, bounds */
3281 BUS_SPACE_MAXADDR, /* lowaddr */
3282 BUS_SPACE_MAXADDR, /* highaddr */
3283 NULL, NULL, /* filter, filterarg */
3286 size, /* maxsegsize */
3288 NULL, /* lockfunc */
3292 device_printf(adapter->dev,
3293 "%s: bus_dma_tag_create failed: %d\n",
3298 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3299 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3301 device_printf(adapter->dev,
3302 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3303 __func__, (uintmax_t)size, error);
3308 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3309 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3310 if (error || dma->dma_paddr == 0) {
3311 device_printf(adapter->dev,
3312 "%s: bus_dmamap_load failed: %d\n",
3320 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3322 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3323 bus_dma_tag_destroy(dma->dma_tag);
3325 dma->dma_tag = NULL;
3331 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
3333 if (dma->dma_tag == NULL)
3335 if (dma->dma_paddr != 0) {
3336 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3337 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3338 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3341 if (dma->dma_vaddr != NULL) {
3342 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3343 dma->dma_vaddr = NULL;
3345 bus_dma_tag_destroy(dma->dma_tag);
3346 dma->dma_tag = NULL;
3350 /*********************************************************************
3352 * Allocate memory for the transmit and receive rings, and then
3353 * the descriptors associated with each, called only once at attach.
3355 **********************************************************************/
3357 igb_allocate_queues(struct adapter *adapter)
3359 device_t dev = adapter->dev;
3360 struct igb_queue *que = NULL;
3361 struct tx_ring *txr = NULL;
3362 struct rx_ring *rxr = NULL;
3363 int rsize, tsize, error = E1000_SUCCESS;
3364 int txconf = 0, rxconf = 0;
3366 /* First allocate the top level queue structs */
3367 if (!(adapter->queues =
3368 (struct igb_queue *) malloc(sizeof(struct igb_queue) *
3369 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3370 device_printf(dev, "Unable to allocate queue memory\n");
3375 /* Next allocate the TX ring struct memory */
3376 if (!(adapter->tx_rings =
3377 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3378 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3379 device_printf(dev, "Unable to allocate TX ring memory\n");
3384 /* Now allocate the RX */
3385 if (!(adapter->rx_rings =
3386 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3387 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3388 device_printf(dev, "Unable to allocate RX ring memory\n");
3393 tsize = roundup2(adapter->num_tx_desc *
3394 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
3396 * Now set up the TX queues, txconf is needed to handle the
3397 * possibility that things fail midcourse and we need to
3398 * undo memory gracefully
3400 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3401 /* Set up some basics */
3402 txr = &adapter->tx_rings[i];
3403 txr->adapter = adapter;
3405 txr->num_desc = adapter->num_tx_desc;
3407 /* Initialize the TX lock */
3408 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3409 device_get_nameunit(dev), txr->me);
3410 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3412 if (igb_dma_malloc(adapter, tsize,
3413 &txr->txdma, BUS_DMA_NOWAIT)) {
3415 "Unable to allocate TX Descriptor memory\n");
3419 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
3420 bzero((void *)txr->tx_base, tsize);
3422 /* Now allocate transmit buffers for the ring */
3423 if (igb_allocate_transmit_buffers(txr)) {
3425 "Critical Failure setting up transmit buffers\n");
3429 #ifndef IGB_LEGACY_TX
3430 /* Allocate a buf ring */
3431 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
3432 M_WAITOK, &txr->tx_mtx);
3437 * Next the RX queues...
3439 rsize = roundup2(adapter->num_rx_desc *
3440 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3441 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3442 rxr = &adapter->rx_rings[i];
3443 rxr->adapter = adapter;
3446 /* Initialize the RX lock */
3447 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3448 device_get_nameunit(dev), txr->me);
3449 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3451 if (igb_dma_malloc(adapter, rsize,
3452 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3454 "Unable to allocate RxDescriptor memory\n");
3458 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
3459 bzero((void *)rxr->rx_base, rsize);
3461 /* Allocate receive buffers for the ring*/
3462 if (igb_allocate_receive_buffers(rxr)) {
3464 "Critical Failure setting up receive buffers\n");
3471 ** Finally set up the queue holding structs
3473 for (int i = 0; i < adapter->num_queues; i++) {
3474 que = &adapter->queues[i];
3475 que->adapter = adapter;
3476 que->txr = &adapter->tx_rings[i];
3477 que->rxr = &adapter->rx_rings[i];
3483 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3484 igb_dma_free(adapter, &rxr->rxdma);
3486 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3487 igb_dma_free(adapter, &txr->txdma);
3488 free(adapter->rx_rings, M_DEVBUF);
3490 #ifndef IGB_LEGACY_TX
3491 buf_ring_free(txr->br, M_DEVBUF);
3493 free(adapter->tx_rings, M_DEVBUF);
3495 free(adapter->queues, M_DEVBUF);
3500 /*********************************************************************
3502 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3503 * the information needed to transmit a packet on the wire. This is
3504 * called only once at attach, setup is done every reset.
3506 **********************************************************************/
3508 igb_allocate_transmit_buffers(struct tx_ring *txr)
3510 struct adapter *adapter = txr->adapter;
3511 device_t dev = adapter->dev;
3512 struct igb_tx_buf *txbuf;
3516 * Setup DMA descriptor areas.
3518 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3519 1, 0, /* alignment, bounds */
3520 BUS_SPACE_MAXADDR, /* lowaddr */
3521 BUS_SPACE_MAXADDR, /* highaddr */
3522 NULL, NULL, /* filter, filterarg */
3523 IGB_TSO_SIZE, /* maxsize */
3524 IGB_MAX_SCATTER, /* nsegments */
3525 PAGE_SIZE, /* maxsegsize */
3527 NULL, /* lockfunc */
3528 NULL, /* lockfuncarg */
3530 device_printf(dev,"Unable to allocate TX DMA tag\n");
3534 if (!(txr->tx_buffers =
3535 (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
3536 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3537 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3542 /* Create the descriptor buffer dma maps */
3543 txbuf = txr->tx_buffers;
3544 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3545 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3547 device_printf(dev, "Unable to create TX DMA map\n");
3554 /* We free all, it handles case where we are in the middle */
3555 igb_free_transmit_structures(adapter);
3559 /*********************************************************************
3561 * Initialize a transmit ring.
3563 **********************************************************************/
3565 igb_setup_transmit_ring(struct tx_ring *txr)
3567 struct adapter *adapter = txr->adapter;
3568 struct igb_tx_buf *txbuf;
3571 struct netmap_adapter *na = NA(adapter->ifp);
3572 struct netmap_slot *slot;
3573 #endif /* DEV_NETMAP */
3575 /* Clear the old descriptor contents */
3578 slot = netmap_reset(na, NR_TX, txr->me, 0);
3579 #endif /* DEV_NETMAP */
3580 bzero((void *)txr->tx_base,
3581 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3583 txr->next_avail_desc = 0;
3584 txr->next_to_clean = 0;
3586 /* Free any existing tx buffers. */
3587 txbuf = txr->tx_buffers;
3588 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3589 if (txbuf->m_head != NULL) {
3590 bus_dmamap_sync(txr->txtag, txbuf->map,
3591 BUS_DMASYNC_POSTWRITE);
3592 bus_dmamap_unload(txr->txtag, txbuf->map);
3593 m_freem(txbuf->m_head);
3594 txbuf->m_head = NULL;
3598 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3599 /* no need to set the address */
3600 netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
3602 #endif /* DEV_NETMAP */
3603 /* clear the watch index */
3607 /* Set number of descriptors available */
3608 txr->tx_avail = adapter->num_tx_desc;
3610 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3611 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3615 /*********************************************************************
3617 * Initialize all transmit rings.
3619 **********************************************************************/
3621 igb_setup_transmit_structures(struct adapter *adapter)
3623 struct tx_ring *txr = adapter->tx_rings;
3625 for (int i = 0; i < adapter->num_queues; i++, txr++)
3626 igb_setup_transmit_ring(txr);
3631 /*********************************************************************
3633 * Enable transmit unit.
3635 **********************************************************************/
3637 igb_initialize_transmit_units(struct adapter *adapter)
3639 struct tx_ring *txr = adapter->tx_rings;
3640 struct e1000_hw *hw = &adapter->hw;
3643 INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3646 /* Setup the Tx Descriptor Rings */
3647 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3648 u64 bus_addr = txr->txdma.dma_paddr;
3650 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3651 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3652 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3653 (uint32_t)(bus_addr >> 32));
3654 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3655 (uint32_t)bus_addr);
3657 /* Setup the HW Tx Head and Tail descriptor pointers */
3658 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3659 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3661 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3662 E1000_READ_REG(hw, E1000_TDBAL(i)),
3663 E1000_READ_REG(hw, E1000_TDLEN(i)));
3665 txr->queue_status = IGB_QUEUE_IDLE;
3667 txdctl |= IGB_TX_PTHRESH;
3668 txdctl |= IGB_TX_HTHRESH << 8;
3669 txdctl |= IGB_TX_WTHRESH << 16;
3670 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3671 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3674 if (adapter->vf_ifp)
3677 e1000_config_collision_dist(hw);
3679 /* Program the Transmit Control Register */
3680 tctl = E1000_READ_REG(hw, E1000_TCTL);
3681 tctl &= ~E1000_TCTL_CT;
3682 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3683 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3685 /* This write will effectively turn on the transmit unit. */
3686 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3689 /*********************************************************************
3691 * Free all transmit rings.
3693 **********************************************************************/
3695 igb_free_transmit_structures(struct adapter *adapter)
3697 struct tx_ring *txr = adapter->tx_rings;
3699 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3701 igb_free_transmit_buffers(txr);
3702 igb_dma_free(adapter, &txr->txdma);
3704 IGB_TX_LOCK_DESTROY(txr);
3706 free(adapter->tx_rings, M_DEVBUF);
3709 /*********************************************************************
3711 * Free transmit ring related data structures.
3713 **********************************************************************/
3715 igb_free_transmit_buffers(struct tx_ring *txr)
3717 struct adapter *adapter = txr->adapter;
3718 struct igb_tx_buf *tx_buffer;
3721 INIT_DEBUGOUT("free_transmit_ring: begin");
3723 if (txr->tx_buffers == NULL)
3726 tx_buffer = txr->tx_buffers;
3727 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3728 if (tx_buffer->m_head != NULL) {
3729 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3730 BUS_DMASYNC_POSTWRITE);
3731 bus_dmamap_unload(txr->txtag,
3733 m_freem(tx_buffer->m_head);
3734 tx_buffer->m_head = NULL;
3735 if (tx_buffer->map != NULL) {
3736 bus_dmamap_destroy(txr->txtag,
3738 tx_buffer->map = NULL;
3740 } else if (tx_buffer->map != NULL) {
3741 bus_dmamap_unload(txr->txtag,
3743 bus_dmamap_destroy(txr->txtag,
3745 tx_buffer->map = NULL;
3748 #ifndef IGB_LEGACY_TX
3749 if (txr->br != NULL)
3750 buf_ring_free(txr->br, M_DEVBUF);
3752 if (txr->tx_buffers != NULL) {
3753 free(txr->tx_buffers, M_DEVBUF);
3754 txr->tx_buffers = NULL;
3756 if (txr->txtag != NULL) {
3757 bus_dma_tag_destroy(txr->txtag);
3763 /**********************************************************************
3765 * Setup work for hardware segmentation offload (TSO) on
3766 * adapters using advanced tx descriptors
3768 **********************************************************************/
3770 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3771 u32 *cmd_type_len, u32 *olinfo_status)
3773 struct adapter *adapter = txr->adapter;
3774 struct e1000_adv_tx_context_desc *TXD;
3775 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3776 u32 mss_l4len_idx = 0, paylen;
3777 u16 vtag = 0, eh_type;
3778 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3779 struct ether_vlan_header *eh;
3781 struct ip6_hdr *ip6;
3790 * Determine where frame payload starts.
3791 * Jump over vlan headers if already present
3793 eh = mtod(mp, struct ether_vlan_header *);
3794 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3795 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3796 eh_type = eh->evl_proto;
3798 ehdrlen = ETHER_HDR_LEN;
3799 eh_type = eh->evl_encap_proto;
3802 switch (ntohs(eh_type)) {
3804 case ETHERTYPE_IPV6:
3805 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3806 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3807 if (ip6->ip6_nxt != IPPROTO_TCP)
3809 ip_hlen = sizeof(struct ip6_hdr);
3810 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3811 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3812 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3813 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3818 ip = (struct ip *)(mp->m_data + ehdrlen);
3819 if (ip->ip_p != IPPROTO_TCP)
3822 ip_hlen = ip->ip_hl << 2;
3823 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3824 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3825 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3826 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3827 /* Tell transmit desc to also do IPv4 checksum. */
3828 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3832 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3833 __func__, ntohs(eh_type));
3837 ctxd = txr->next_avail_desc;
3838 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3840 tcp_hlen = th->th_off << 2;
3842 /* This is used in the transmit desc in encap */
3843 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3845 /* VLAN MACLEN IPLEN */
3846 if (mp->m_flags & M_VLANTAG) {
3847 vtag = htole16(mp->m_pkthdr.ether_vtag);
3848 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3851 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3852 vlan_macip_lens |= ip_hlen;
3853 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3855 /* ADV DTYPE TUCMD */
3856 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3857 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3858 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3861 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3862 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3863 /* 82575 needs the queue index added */
3864 if (adapter->hw.mac.type == e1000_82575)
3865 mss_l4len_idx |= txr->me << 4;
3866 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3868 TXD->seqnum_seed = htole32(0);
3870 if (++ctxd == txr->num_desc)
3874 txr->next_avail_desc = ctxd;
3875 *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3876 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3877 *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
3882 /*********************************************************************
3884 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3886 **********************************************************************/
3889 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3890 u32 *cmd_type_len, u32 *olinfo_status)
3892 struct e1000_adv_tx_context_desc *TXD;
3893 struct adapter *adapter = txr->adapter;
3894 struct ether_vlan_header *eh;
3896 struct ip6_hdr *ip6;
3897 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
3898 int ehdrlen, ip_hlen = 0;
3902 int ctxd = txr->next_avail_desc;
3905 /* First check if TSO is to be used */
3906 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3907 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3909 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3912 /* Indicate the whole packet as payload when not doing TSO */
3913 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
3915 /* Now ready a context descriptor */
3916 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3919 ** In advanced descriptors the vlan tag must
3920 ** be placed into the context descriptor. Hence
3921 ** we need to make one even if not doing offloads.
3923 if (mp->m_flags & M_VLANTAG) {
3924 vtag = htole16(mp->m_pkthdr.ether_vtag);
3925 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3926 } else if (offload == FALSE) /* ... no offload to do */
3930 * Determine where frame payload starts.
3931 * Jump over vlan headers if already present,
3932 * helpful for QinQ too.
3934 eh = mtod(mp, struct ether_vlan_header *);
3935 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3936 etype = ntohs(eh->evl_proto);
3937 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3939 etype = ntohs(eh->evl_encap_proto);
3940 ehdrlen = ETHER_HDR_LEN;
3943 /* Set the ether header length */
3944 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3948 ip = (struct ip *)(mp->m_data + ehdrlen);
3949 ip_hlen = ip->ip_hl << 2;
3951 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3953 case ETHERTYPE_IPV6:
3954 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3955 ip_hlen = sizeof(struct ip6_hdr);
3956 /* XXX-BZ this will go badly in case of ext hdrs. */
3957 ipproto = ip6->ip6_nxt;
3958 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3965 vlan_macip_lens |= ip_hlen;
3966 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3970 #if __FreeBSD_version >= 1000000
3971 if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))
3973 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3975 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3978 #if __FreeBSD_version >= 1000000
3979 if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP))
3981 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3983 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3986 #if __FreeBSD_version >= 800000
3988 #if __FreeBSD_version >= 1000000
3989 if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP))
3991 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3993 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4001 if (offload) /* For the TX descriptor setup */
4002 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4004 /* 82575 needs the queue index added */
4005 if (adapter->hw.mac.type == e1000_82575)
4006 mss_l4len_idx = txr->me << 4;
4008 /* Now copy bits into descriptor */
4009 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
4010 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
4011 TXD->seqnum_seed = htole32(0);
4012 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
4014 /* We've consumed the first desc, adjust counters */
4015 if (++ctxd == txr->num_desc)
4017 txr->next_avail_desc = ctxd;
4023 /**********************************************************************
4025 * Examine each tx_buffer in the used queue. If the hardware is done
4026 * processing the packet then free associated resources. The
4027 * tx_buffer is put back on the free queue.
4029 * TRUE return means there's work in the ring to clean, FALSE its empty.
4030 **********************************************************************/
4032 igb_txeof(struct tx_ring *txr)
4034 struct adapter *adapter = txr->adapter;
4036 struct ifnet *ifp = adapter->ifp;
4037 #endif /* DEV_NETMAP */
4038 u32 work, processed = 0;
4039 int limit = adapter->tx_process_limit;
4040 struct igb_tx_buf *buf;
4041 union e1000_adv_tx_desc *txd;
4043 mtx_assert(&txr->tx_mtx, MA_OWNED);
4046 if (netmap_tx_irq(ifp, txr->me))
4048 #endif /* DEV_NETMAP */
4050 if (txr->tx_avail == txr->num_desc) {
4051 txr->queue_status = IGB_QUEUE_IDLE;
4055 /* Get work starting point */
4056 work = txr->next_to_clean;
4057 buf = &txr->tx_buffers[work];
4058 txd = &txr->tx_base[work];
4059 work -= txr->num_desc; /* The distance to ring end */
4060 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4061 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4063 union e1000_adv_tx_desc *eop = buf->eop;
4064 if (eop == NULL) /* No work */
4067 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
4068 break; /* I/O not complete */
4072 buf->m_head->m_pkthdr.len;
4073 bus_dmamap_sync(txr->txtag,
4075 BUS_DMASYNC_POSTWRITE);
4076 bus_dmamap_unload(txr->txtag,
4078 m_freem(buf->m_head);
4084 /* We clean the range if multi segment */
4085 while (txd != eop) {
4089 /* wrap the ring? */
4090 if (__predict_false(!work)) {
4091 work -= txr->num_desc;
4092 buf = txr->tx_buffers;
4097 buf->m_head->m_pkthdr.len;
4098 bus_dmamap_sync(txr->txtag,
4100 BUS_DMASYNC_POSTWRITE);
4101 bus_dmamap_unload(txr->txtag,
4103 m_freem(buf->m_head);
4112 txr->watchdog_time = ticks;
4114 /* Try the next packet */
4118 /* reset with a wrap */
4119 if (__predict_false(!work)) {
4120 work -= txr->num_desc;
4121 buf = txr->tx_buffers;
4125 } while (__predict_true(--limit));
4127 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4128 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4130 work += txr->num_desc;
4131 txr->next_to_clean = work;
4134 ** Watchdog calculation, we know there's
4135 ** work outstanding or the first return
4136 ** would have been taken, so none processed
4137 ** for too long indicates a hang.
4139 if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
4140 txr->queue_status |= IGB_QUEUE_HUNG;
4142 if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
4143 txr->queue_status &= ~IGB_QUEUE_DEPLETED;
4145 if (txr->tx_avail == txr->num_desc) {
4146 txr->queue_status = IGB_QUEUE_IDLE;
4153 /*********************************************************************
4155 * Refresh mbuf buffers for RX descriptor rings
4156 * - now keeps its own state so discards due to resource
4157 * exhaustion are unnecessary, if an mbuf cannot be obtained
4158 * it just returns, keeping its placeholder, thus it can simply
4159 * be recalled to try again.
4161 **********************************************************************/
4163 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
4165 struct adapter *adapter = rxr->adapter;
4166 bus_dma_segment_t hseg[1];
4167 bus_dma_segment_t pseg[1];
4168 struct igb_rx_buf *rxbuf;
4169 struct mbuf *mh, *mp;
4170 int i, j, nsegs, error;
4171 bool refreshed = FALSE;
4173 i = j = rxr->next_to_refresh;
4175 ** Get one descriptor beyond
4176 ** our work mark to control
4179 if (++j == adapter->num_rx_desc)
4182 while (j != limit) {
4183 rxbuf = &rxr->rx_buffers[i];
4184 /* No hdr mbuf used with header split off */
4185 if (rxr->hdr_split == FALSE)
4187 if (rxbuf->m_head == NULL) {
4188 mh = m_gethdr(M_NOWAIT, MT_DATA);
4194 mh->m_pkthdr.len = mh->m_len = MHLEN;
4196 mh->m_flags |= M_PKTHDR;
4197 /* Get the memory mapping */
4198 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4199 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
4201 printf("Refresh mbufs: hdr dmamap load"
4202 " failure - %d\n", error);
4204 rxbuf->m_head = NULL;
4208 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4209 BUS_DMASYNC_PREREAD);
4210 rxr->rx_base[i].read.hdr_addr =
4211 htole64(hseg[0].ds_addr);
4213 if (rxbuf->m_pack == NULL) {
4214 mp = m_getjcl(M_NOWAIT, MT_DATA,
4215 M_PKTHDR, adapter->rx_mbuf_sz);
4221 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4222 /* Get the memory mapping */
4223 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4224 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
4226 printf("Refresh mbufs: payload dmamap load"
4227 " failure - %d\n", error);
4229 rxbuf->m_pack = NULL;
4233 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4234 BUS_DMASYNC_PREREAD);
4235 rxr->rx_base[i].read.pkt_addr =
4236 htole64(pseg[0].ds_addr);
4237 refreshed = TRUE; /* I feel wefreshed :) */
4239 i = j; /* our next is precalculated */
4240 rxr->next_to_refresh = i;
4241 if (++j == adapter->num_rx_desc)
4245 if (refreshed) /* update tail */
4246 E1000_WRITE_REG(&adapter->hw,
4247 E1000_RDT(rxr->me), rxr->next_to_refresh);
4252 /*********************************************************************
4254 * Allocate memory for rx_buffer structures. Since we use one
4255 * rx_buffer per received packet, the maximum number of rx_buffer's
4256 * that we'll need is equal to the number of receive descriptors
4257 * that we've allocated.
4259 **********************************************************************/
4261 igb_allocate_receive_buffers(struct rx_ring *rxr)
4263 struct adapter *adapter = rxr->adapter;
4264 device_t dev = adapter->dev;
4265 struct igb_rx_buf *rxbuf;
4266 int i, bsize, error;
4268 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
4269 if (!(rxr->rx_buffers =
4270 (struct igb_rx_buf *) malloc(bsize,
4271 M_DEVBUF, M_NOWAIT | M_ZERO))) {
4272 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4277 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4278 1, 0, /* alignment, bounds */
4279 BUS_SPACE_MAXADDR, /* lowaddr */
4280 BUS_SPACE_MAXADDR, /* highaddr */
4281 NULL, NULL, /* filter, filterarg */
4282 MSIZE, /* maxsize */
4284 MSIZE, /* maxsegsize */
4286 NULL, /* lockfunc */
4287 NULL, /* lockfuncarg */
4289 device_printf(dev, "Unable to create RX DMA tag\n");
4293 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
4294 1, 0, /* alignment, bounds */
4295 BUS_SPACE_MAXADDR, /* lowaddr */
4296 BUS_SPACE_MAXADDR, /* highaddr */
4297 NULL, NULL, /* filter, filterarg */
4298 MJUM9BYTES, /* maxsize */
4300 MJUM9BYTES, /* maxsegsize */
4302 NULL, /* lockfunc */
4303 NULL, /* lockfuncarg */
4305 device_printf(dev, "Unable to create RX payload DMA tag\n");
4309 for (i = 0; i < adapter->num_rx_desc; i++) {
4310 rxbuf = &rxr->rx_buffers[i];
4311 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
4314 "Unable to create RX head DMA maps\n");
4317 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
4320 "Unable to create RX packet DMA maps\n");
4328 /* Frees all, but can handle partial completion */
4329 igb_free_receive_structures(adapter);
4335 igb_free_receive_ring(struct rx_ring *rxr)
4337 struct adapter *adapter = rxr->adapter;
4338 struct igb_rx_buf *rxbuf;
4341 for (int i = 0; i < adapter->num_rx_desc; i++) {
4342 rxbuf = &rxr->rx_buffers[i];
4343 if (rxbuf->m_head != NULL) {
4344 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4345 BUS_DMASYNC_POSTREAD);
4346 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4347 rxbuf->m_head->m_flags |= M_PKTHDR;
4348 m_freem(rxbuf->m_head);
4350 if (rxbuf->m_pack != NULL) {
4351 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4352 BUS_DMASYNC_POSTREAD);
4353 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4354 rxbuf->m_pack->m_flags |= M_PKTHDR;
4355 m_freem(rxbuf->m_pack);
4357 rxbuf->m_head = NULL;
4358 rxbuf->m_pack = NULL;
4363 /*********************************************************************
4365 * Initialize a receive ring and its buffers.
4367 **********************************************************************/
4369 igb_setup_receive_ring(struct rx_ring *rxr)
4371 struct adapter *adapter;
4374 struct igb_rx_buf *rxbuf;
4375 bus_dma_segment_t pseg[1], hseg[1];
4376 struct lro_ctrl *lro = &rxr->lro;
4377 int rsize, nsegs, error = 0;
4379 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4380 struct netmap_slot *slot;
4381 #endif /* DEV_NETMAP */
4383 adapter = rxr->adapter;
4387 /* Clear the ring contents */
4390 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4391 #endif /* DEV_NETMAP */
4392 rsize = roundup2(adapter->num_rx_desc *
4393 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
4394 bzero((void *)rxr->rx_base, rsize);
4397 ** Free current RX buffer structures and their mbufs
4399 igb_free_receive_ring(rxr);
4401 /* Configure for header split? */
4402 if (igb_header_split)
4403 rxr->hdr_split = TRUE;
4405 /* Now replenish the ring mbufs */
4406 for (int j = 0; j < adapter->num_rx_desc; ++j) {
4407 struct mbuf *mh, *mp;
4409 rxbuf = &rxr->rx_buffers[j];
4412 /* slot sj is mapped to the j-th NIC-ring entry */
4413 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4417 addr = PNMB(na, slot + sj, &paddr);
4418 netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
4419 /* Update descriptor */
4420 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4423 #endif /* DEV_NETMAP */
4424 if (rxr->hdr_split == FALSE)
4427 /* First the header */
4428 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4429 if (rxbuf->m_head == NULL) {
4433 m_adj(rxbuf->m_head, ETHER_ALIGN);
4435 mh->m_len = mh->m_pkthdr.len = MHLEN;
4436 mh->m_flags |= M_PKTHDR;
4437 /* Get the memory mapping */
4438 error = bus_dmamap_load_mbuf_sg(rxr->htag,
4439 rxbuf->hmap, rxbuf->m_head, hseg,
4440 &nsegs, BUS_DMA_NOWAIT);
4441 if (error != 0) /* Nothing elegant to do here */
4443 bus_dmamap_sync(rxr->htag,
4444 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4445 /* Update descriptor */
4446 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4449 /* Now the payload cluster */
4450 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4451 M_PKTHDR, adapter->rx_mbuf_sz);
4452 if (rxbuf->m_pack == NULL) {
4457 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4458 /* Get the memory mapping */
4459 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4460 rxbuf->pmap, mp, pseg,
4461 &nsegs, BUS_DMA_NOWAIT);
4464 bus_dmamap_sync(rxr->ptag,
4465 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4466 /* Update descriptor */
4467 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4470 /* Setup our descriptor indices */
4471 rxr->next_to_check = 0;
4472 rxr->next_to_refresh = adapter->num_rx_desc - 1;
4473 rxr->lro_enabled = FALSE;
4474 rxr->rx_split_packets = 0;
4480 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4481 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4484 ** Now set up the LRO interface, we
4485 ** also only do head split when LRO
4486 ** is enabled, since so often they
4487 ** are undesirable in similar setups.
4489 if (ifp->if_capenable & IFCAP_LRO) {
4490 error = tcp_lro_init(lro);
4492 device_printf(dev, "LRO Initialization failed!\n");
4495 INIT_DEBUGOUT("RX LRO Initialized\n");
4496 rxr->lro_enabled = TRUE;
4497 lro->ifp = adapter->ifp;
4504 igb_free_receive_ring(rxr);
4510 /*********************************************************************
4512 * Initialize all receive rings.
4514 **********************************************************************/
4516 igb_setup_receive_structures(struct adapter *adapter)
4518 struct rx_ring *rxr = adapter->rx_rings;
4521 for (i = 0; i < adapter->num_queues; i++, rxr++)
4522 if (igb_setup_receive_ring(rxr))
4528 * Free RX buffers allocated so far, we will only handle
4529 * the rings that completed, the failing case will have
4530 * cleaned up for itself. 'i' is the endpoint.
4532 for (int j = 0; j < i; ++j) {
4533 rxr = &adapter->rx_rings[j];
4535 igb_free_receive_ring(rxr);
4543 * Initialise the RSS mapping for NICs that support multiple transmit/
4547 igb_initialise_rss_mapping(struct adapter *adapter)
4549 struct e1000_hw *hw = &adapter->hw;
4553 u32 rss_key[10], mrqc, shift = 0;
4556 if (adapter->hw.mac.type == e1000_82575)
4560 * The redirection table controls which destination
4561 * queue each bucket redirects traffic to.
4562 * Each DWORD represents four queues, with the LSB
4563 * being the first queue in the DWORD.
4565 * This just allocates buckets to queues using round-robin
4568 * NOTE: It Just Happens to line up with the default
4569 * RSS allocation method.
4572 /* Warning FM follows */
4574 for (i = 0; i < 128; i++) {
4576 queue_id = rss_get_indirection_to_bucket(i);
4578 * If we have more queues than buckets, we'll
4579 * end up mapping buckets to a subset of the
4582 * If we have more buckets than queues, we'll
4583 * end up instead assigning multiple buckets
4586 * Both are suboptimal, but we need to handle
4587 * the case so we don't go out of bounds
4588 * indexing arrays and such.
4590 queue_id = queue_id % adapter->num_queues;
4592 queue_id = (i % adapter->num_queues);
4594 /* Adjust if required */
4595 queue_id = queue_id << shift;
4598 * The low 8 bits are for hash value (n+0);
4599 * The next 8 bits are for hash value (n+1), etc.
4602 reta = reta | ( ((uint32_t) queue_id) << 24);
4604 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
4609 /* Now fill in hash table */
4612 * MRQC: Multiple Receive Queues Command
4613 * Set queuing to RSS control, number depends on the device.
4615 mrqc = E1000_MRQC_ENABLE_RSS_8Q;
4618 /* XXX ew typecasting */
4619 rss_getkey((uint8_t *) &rss_key);
4621 arc4rand(&rss_key, sizeof(rss_key), 0);
4623 for (i = 0; i < 10; i++)
4624 E1000_WRITE_REG_ARRAY(hw,
4625 E1000_RSSRK(0), i, rss_key[i]);
4628 * Configure the RSS fields to hash upon.
4630 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
4631 E1000_MRQC_RSS_FIELD_IPV4_TCP);
4632 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
4633 E1000_MRQC_RSS_FIELD_IPV6_TCP);
4634 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
4635 E1000_MRQC_RSS_FIELD_IPV6_UDP);
4636 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
4637 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
4639 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
4642 /*********************************************************************
4644 * Enable receive unit.
4646 **********************************************************************/
4648 igb_initialize_receive_units(struct adapter *adapter)
4650 struct rx_ring *rxr = adapter->rx_rings;
4651 struct ifnet *ifp = adapter->ifp;
4652 struct e1000_hw *hw = &adapter->hw;
4653 u32 rctl, rxcsum, psize, srrctl = 0;
4655 INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
4658 * Make sure receives are disabled while setting
4659 * up the descriptor ring
4661 rctl = E1000_READ_REG(hw, E1000_RCTL);
4662 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4665 ** Set up for header split
4667 if (igb_header_split) {
4668 /* Use a standard mbuf for the header */
4669 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4670 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4672 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4675 ** Set up for jumbo frames
4677 if (ifp->if_mtu > ETHERMTU) {
4678 rctl |= E1000_RCTL_LPE;
4679 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
4680 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4681 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4682 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
4683 srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4684 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4686 /* Set maximum packet len */
4687 psize = adapter->max_frame_size;
4688 /* are we on a vlan? */
4689 if (adapter->ifp->if_vlantrunk != NULL)
4690 psize += VLAN_TAG_SIZE;
4691 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
4693 rctl &= ~E1000_RCTL_LPE;
4694 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4695 rctl |= E1000_RCTL_SZ_2048;
4699 * If TX flow control is disabled and there's >1 queue defined,
4702 * This drops frames rather than hanging the RX MAC for all queues.
4704 if ((adapter->num_queues > 1) &&
4705 (adapter->fc == e1000_fc_none ||
4706 adapter->fc == e1000_fc_rx_pause)) {
4707 srrctl |= E1000_SRRCTL_DROP_EN;
4710 /* Setup the Base and Length of the Rx Descriptor Rings */
4711 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4712 u64 bus_addr = rxr->rxdma.dma_paddr;
4715 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4716 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4717 E1000_WRITE_REG(hw, E1000_RDBAH(i),
4718 (uint32_t)(bus_addr >> 32));
4719 E1000_WRITE_REG(hw, E1000_RDBAL(i),
4720 (uint32_t)bus_addr);
4721 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
4722 /* Enable this Queue */
4723 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
4724 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4725 rxdctl &= 0xFFF00000;
4726 rxdctl |= IGB_RX_PTHRESH;
4727 rxdctl |= IGB_RX_HTHRESH << 8;
4728 rxdctl |= IGB_RX_WTHRESH << 16;
4729 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
4733 ** Setup for RX MultiQueue
4735 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4736 if (adapter->num_queues >1) {
4739 igb_initialise_rss_mapping(adapter);
4742 ** NOTE: Receive Full-Packet Checksum Offload
4743 ** is mutually exclusive with Multiqueue. However
4744 ** this is not the same as TCP/IP checksums which
4747 rxcsum |= E1000_RXCSUM_PCSD;
4748 #if __FreeBSD_version >= 800000
4749 /* For SCTP Offload */
4750 if ((hw->mac.type != e1000_82575) &&
4751 (ifp->if_capenable & IFCAP_RXCSUM))
4752 rxcsum |= E1000_RXCSUM_CRCOFL;
4756 if (ifp->if_capenable & IFCAP_RXCSUM) {
4757 rxcsum |= E1000_RXCSUM_IPPCSE;
4758 #if __FreeBSD_version >= 800000
4759 if (adapter->hw.mac.type != e1000_82575)
4760 rxcsum |= E1000_RXCSUM_CRCOFL;
4763 rxcsum &= ~E1000_RXCSUM_TUOFL;
4765 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4767 /* Setup the Receive Control Register */
4768 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4769 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4770 E1000_RCTL_RDMTS_HALF |
4771 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4772 /* Strip CRC bytes. */
4773 rctl |= E1000_RCTL_SECRC;
4774 /* Make sure VLAN Filters are off */
4775 rctl &= ~E1000_RCTL_VFE;
4776 /* Don't store bad packets */
4777 rctl &= ~E1000_RCTL_SBP;
4779 /* Enable Receives */
4780 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4783 * Setup the HW Rx Head and Tail Descriptor Pointers
4784 * - needs to be after enable
4786 for (int i = 0; i < adapter->num_queues; i++) {
4787 rxr = &adapter->rx_rings[i];
4788 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
4791 * an init() while a netmap client is active must
4792 * preserve the rx buffers passed to userspace.
4793 * In this driver it means we adjust RDT to
4794 * something different from next_to_refresh
4795 * (which is not used in netmap mode).
4797 if (ifp->if_capenable & IFCAP_NETMAP) {
4798 struct netmap_adapter *na = NA(adapter->ifp);
4799 struct netmap_kring *kring = &na->rx_rings[i];
4800 int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
4802 if (t >= adapter->num_rx_desc)
4803 t -= adapter->num_rx_desc;
4805 t += adapter->num_rx_desc;
4806 E1000_WRITE_REG(hw, E1000_RDT(i), t);
4808 #endif /* DEV_NETMAP */
4809 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
4814 /*********************************************************************
4816 * Free receive rings.
4818 **********************************************************************/
4820 igb_free_receive_structures(struct adapter *adapter)
4822 struct rx_ring *rxr = adapter->rx_rings;
4824 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4825 struct lro_ctrl *lro = &rxr->lro;
4826 igb_free_receive_buffers(rxr);
4828 igb_dma_free(adapter, &rxr->rxdma);
4831 free(adapter->rx_rings, M_DEVBUF);
4834 /*********************************************************************
4836 * Free receive ring data structures.
4838 **********************************************************************/
4840 igb_free_receive_buffers(struct rx_ring *rxr)
4842 struct adapter *adapter = rxr->adapter;
4843 struct igb_rx_buf *rxbuf;
4846 INIT_DEBUGOUT("free_receive_structures: begin");
4848 /* Cleanup any existing buffers */
4849 if (rxr->rx_buffers != NULL) {
4850 for (i = 0; i < adapter->num_rx_desc; i++) {
4851 rxbuf = &rxr->rx_buffers[i];
4852 if (rxbuf->m_head != NULL) {
4853 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4854 BUS_DMASYNC_POSTREAD);
4855 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4856 rxbuf->m_head->m_flags |= M_PKTHDR;
4857 m_freem(rxbuf->m_head);
4859 if (rxbuf->m_pack != NULL) {
4860 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4861 BUS_DMASYNC_POSTREAD);
4862 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4863 rxbuf->m_pack->m_flags |= M_PKTHDR;
4864 m_freem(rxbuf->m_pack);
4866 rxbuf->m_head = NULL;
4867 rxbuf->m_pack = NULL;
4868 if (rxbuf->hmap != NULL) {
4869 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4872 if (rxbuf->pmap != NULL) {
4873 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4877 if (rxr->rx_buffers != NULL) {
4878 free(rxr->rx_buffers, M_DEVBUF);
4879 rxr->rx_buffers = NULL;
4883 if (rxr->htag != NULL) {
4884 bus_dma_tag_destroy(rxr->htag);
4887 if (rxr->ptag != NULL) {
4888 bus_dma_tag_destroy(rxr->ptag);
4893 static __inline void
4894 igb_rx_discard(struct rx_ring *rxr, int i)
4896 struct igb_rx_buf *rbuf;
4898 rbuf = &rxr->rx_buffers[i];
4900 /* Partially received? Free the chain */
4901 if (rxr->fmp != NULL) {
4902 rxr->fmp->m_flags |= M_PKTHDR;
4909 ** With advanced descriptors the writeback
4910 ** clobbers the buffer addrs, so its easier
4911 ** to just free the existing mbufs and take
4912 ** the normal refresh path to get new buffers
4916 m_free(rbuf->m_head);
4917 rbuf->m_head = NULL;
4918 bus_dmamap_unload(rxr->htag, rbuf->hmap);
4922 m_free(rbuf->m_pack);
4923 rbuf->m_pack = NULL;
4924 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
4930 static __inline void
4931 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4935 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4936 * should be computed by hardware. Also it should not have VLAN tag in
4939 if (rxr->lro_enabled &&
4940 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4941 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4942 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4943 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4944 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4945 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4947 * Send to the stack if:
4948 ** - LRO not enabled, or
4949 ** - no LRO resources, or
4950 ** - lro enqueue fails
4952 if (rxr->lro.lro_cnt != 0)
4953 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4957 (*ifp->if_input)(ifp, m);
4961 /*********************************************************************
4963 * This routine executes in interrupt context. It replenishes
4964 * the mbufs in the descriptor and sends data which has been
4965 * dma'ed into host memory to upper layer.
4967 * We loop at most count times if count is > 0, or until done if
4970 * Return TRUE if more to clean, FALSE otherwise
4971 *********************************************************************/
4973 igb_rxeof(struct igb_queue *que, int count, int *done)
4975 struct adapter *adapter = que->adapter;
4976 struct rx_ring *rxr = que->rxr;
4977 struct ifnet *ifp = adapter->ifp;
4978 struct lro_ctrl *lro = &rxr->lro;
4979 int i, processed = 0, rxdone = 0;
4980 u32 ptype, staterr = 0;
4981 union e1000_adv_rx_desc *cur;
4984 /* Sync the ring. */
4985 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4986 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4989 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4993 #endif /* DEV_NETMAP */
4995 /* Main clean loop */
4996 for (i = rxr->next_to_check; count != 0;) {
4997 struct mbuf *sendmp, *mh, *mp;
4998 struct igb_rx_buf *rxbuf;
4999 u16 hlen, plen, hdr, vtag, pkt_info;
5002 cur = &rxr->rx_base[i];
5003 staterr = le32toh(cur->wb.upper.status_error);
5004 if ((staterr & E1000_RXD_STAT_DD) == 0)
5006 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
5009 sendmp = mh = mp = NULL;
5010 cur->wb.upper.status_error = 0;
5011 rxbuf = &rxr->rx_buffers[i];
5012 plen = le16toh(cur->wb.upper.length);
5013 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
5014 if (((adapter->hw.mac.type == e1000_i350) ||
5015 (adapter->hw.mac.type == e1000_i354)) &&
5016 (staterr & E1000_RXDEXT_STATERR_LB))
5017 vtag = be16toh(cur->wb.upper.vlan);
5019 vtag = le16toh(cur->wb.upper.vlan);
5020 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
5021 pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
5022 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
5025 * Free the frame (all segments) if we're at EOP and
5028 * The datasheet states that EOP + status is only valid for
5029 * the final segment in a multi-segment frame.
5031 if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
5032 adapter->dropped_pkts++;
5033 ++rxr->rx_discarded;
5034 igb_rx_discard(rxr, i);
5039 ** The way the hardware is configured to
5040 ** split, it will ONLY use the header buffer
5041 ** when header split is enabled, otherwise we
5042 ** get normal behavior, ie, both header and
5043 ** payload are DMA'd into the payload buffer.
5045 ** The fmp test is to catch the case where a
5046 ** packet spans multiple descriptors, in that
5047 ** case only the first header is valid.
5049 if (rxr->hdr_split && rxr->fmp == NULL) {
5050 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
5051 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
5052 E1000_RXDADV_HDRBUFLEN_SHIFT;
5053 if (hlen > IGB_HDR_BUF)
5055 mh = rxr->rx_buffers[i].m_head;
5057 /* clear buf pointer for refresh */
5058 rxbuf->m_head = NULL;
5060 ** Get the payload length, this
5061 ** could be zero if its a small
5065 mp = rxr->rx_buffers[i].m_pack;
5068 /* clear buf pointer */
5069 rxbuf->m_pack = NULL;
5070 rxr->rx_split_packets++;
5074 ** Either no header split, or a
5075 ** secondary piece of a fragmented
5078 mh = rxr->rx_buffers[i].m_pack;
5080 /* clear buf info for refresh */
5081 rxbuf->m_pack = NULL;
5083 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
5085 ++processed; /* So we know when to refresh */
5087 /* Initial frame - setup */
5088 if (rxr->fmp == NULL) {
5089 mh->m_pkthdr.len = mh->m_len;
5090 /* Save the head of the chain */
5094 /* Add payload if split */
5095 mh->m_pkthdr.len += mp->m_len;
5096 rxr->lmp = mh->m_next;
5099 /* Chain mbuf's together */
5100 rxr->lmp->m_next = mh;
5101 rxr->lmp = rxr->lmp->m_next;
5102 rxr->fmp->m_pkthdr.len += mh->m_len;
5106 rxr->fmp->m_pkthdr.rcvif = ifp;
5108 /* capture data for AIM */
5110 rxr->bytes += rxr->fmp->m_pkthdr.len;
5111 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
5113 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
5114 igb_rx_checksum(staterr, rxr->fmp, ptype);
5116 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
5117 (staterr & E1000_RXD_STAT_VP) != 0) {
5118 rxr->fmp->m_pkthdr.ether_vtag = vtag;
5119 rxr->fmp->m_flags |= M_VLANTAG;
5123 * In case of multiqueue, we have RXCSUM.PCSD bit set
5124 * and never cleared. This means we have RSS hash
5125 * available to be used.
5127 if (adapter->num_queues > 1) {
5128 rxr->fmp->m_pkthdr.flowid =
5129 le32toh(cur->wb.lower.hi_dword.rss);
5130 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
5131 case E1000_RXDADV_RSSTYPE_IPV4_TCP:
5132 M_HASHTYPE_SET(rxr->fmp,
5133 M_HASHTYPE_RSS_TCP_IPV4);
5135 case E1000_RXDADV_RSSTYPE_IPV4:
5136 M_HASHTYPE_SET(rxr->fmp,
5137 M_HASHTYPE_RSS_IPV4);
5139 case E1000_RXDADV_RSSTYPE_IPV6_TCP:
5140 M_HASHTYPE_SET(rxr->fmp,
5141 M_HASHTYPE_RSS_TCP_IPV6);
5143 case E1000_RXDADV_RSSTYPE_IPV6_EX:
5144 M_HASHTYPE_SET(rxr->fmp,
5145 M_HASHTYPE_RSS_IPV6_EX);
5147 case E1000_RXDADV_RSSTYPE_IPV6:
5148 M_HASHTYPE_SET(rxr->fmp,
5149 M_HASHTYPE_RSS_IPV6);
5151 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
5152 M_HASHTYPE_SET(rxr->fmp,
5153 M_HASHTYPE_RSS_TCP_IPV6_EX);
5156 /* XXX fallthrough */
5157 M_HASHTYPE_SET(rxr->fmp,
5158 M_HASHTYPE_OPAQUE_HASH);
5161 #ifndef IGB_LEGACY_TX
5162 rxr->fmp->m_pkthdr.flowid = que->msix;
5163 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
5167 /* Make sure to set M_PKTHDR. */
5168 sendmp->m_flags |= M_PKTHDR;
5174 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
5175 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5177 /* Advance our pointers to the next descriptor. */
5178 if (++i == adapter->num_rx_desc)
5181 ** Send to the stack or LRO
5183 if (sendmp != NULL) {
5184 rxr->next_to_check = i;
5185 igb_rx_input(rxr, ifp, sendmp, ptype);
5186 i = rxr->next_to_check;
5190 /* Every 8 descriptors we go to refresh mbufs */
5191 if (processed == 8) {
5192 igb_refresh_mbufs(rxr, i);
5197 /* Catch any remainders */
5198 if (igb_rx_unrefreshed(rxr))
5199 igb_refresh_mbufs(rxr, i);
5201 rxr->next_to_check = i;
5204 * Flush any outstanding LRO work
5206 tcp_lro_flush_all(lro);
5212 return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
5215 /*********************************************************************
5217 * Verify that the hardware indicated that the checksum is valid.
5218 * Inform the stack about the status of checksum so that stack
5219 * doesn't spend time verifying the checksum.
5221 *********************************************************************/
5223 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
5225 u16 status = (u16)staterr;
5226 u8 errors = (u8) (staterr >> 24);
5229 /* Ignore Checksum bit is set */
5230 if (status & E1000_RXD_STAT_IXSM) {
5231 mp->m_pkthdr.csum_flags = 0;
5235 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
5236 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
5240 if (status & E1000_RXD_STAT_IPCS) {
5242 if (!(errors & E1000_RXD_ERR_IPE)) {
5243 /* IP Checksum Good */
5244 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
5245 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
5247 mp->m_pkthdr.csum_flags = 0;
5250 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
5251 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5252 #if __FreeBSD_version >= 800000
5253 if (sctp) /* reassign */
5254 type = CSUM_SCTP_VALID;
5257 if (!(errors & E1000_RXD_ERR_TCPE)) {
5258 mp->m_pkthdr.csum_flags |= type;
5260 mp->m_pkthdr.csum_data = htons(0xffff);
5267 * This routine is run via an vlan
5271 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5273 struct adapter *adapter = ifp->if_softc;
5276 if (ifp->if_softc != arg) /* Not our event */
5279 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5282 IGB_CORE_LOCK(adapter);
5283 index = (vtag >> 5) & 0x7F;
5285 adapter->shadow_vfta[index] |= (1 << bit);
5286 ++adapter->num_vlans;
5287 /* Change hw filter setting */
5288 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5289 igb_setup_vlan_hw_support(adapter);
5290 IGB_CORE_UNLOCK(adapter);
5294 * This routine is run via an vlan
5298 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
5300 struct adapter *adapter = ifp->if_softc;
5303 if (ifp->if_softc != arg)
5306 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5309 IGB_CORE_LOCK(adapter);
5310 index = (vtag >> 5) & 0x7F;
5312 adapter->shadow_vfta[index] &= ~(1 << bit);
5313 --adapter->num_vlans;
5314 /* Change hw filter setting */
5315 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
5316 igb_setup_vlan_hw_support(adapter);
5317 IGB_CORE_UNLOCK(adapter);
5321 igb_setup_vlan_hw_support(struct adapter *adapter)
5323 struct e1000_hw *hw = &adapter->hw;
5324 struct ifnet *ifp = adapter->ifp;
5327 if (adapter->vf_ifp) {
5328 e1000_rlpml_set_vf(hw,
5329 adapter->max_frame_size + VLAN_TAG_SIZE);
5333 reg = E1000_READ_REG(hw, E1000_CTRL);
5334 reg |= E1000_CTRL_VME;
5335 E1000_WRITE_REG(hw, E1000_CTRL, reg);
5337 /* Enable the Filter Table */
5338 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
5339 reg = E1000_READ_REG(hw, E1000_RCTL);
5340 reg &= ~E1000_RCTL_CFIEN;
5341 reg |= E1000_RCTL_VFE;
5342 E1000_WRITE_REG(hw, E1000_RCTL, reg);
5345 /* Update the frame size */
5346 E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
5347 adapter->max_frame_size + VLAN_TAG_SIZE);
5349 /* Don't bother with table if no vlans */
5350 if ((adapter->num_vlans == 0) ||
5351 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
5354 ** A soft reset zero's out the VFTA, so
5355 ** we need to repopulate it now.
5357 for (int i = 0; i < IGB_VFTA_SIZE; i++)
5358 if (adapter->shadow_vfta[i] != 0) {
5359 if (adapter->vf_ifp)
5360 e1000_vfta_set_vf(hw,
5361 adapter->shadow_vfta[i], TRUE);
5363 e1000_write_vfta(hw,
5364 i, adapter->shadow_vfta[i]);
5369 igb_enable_intr(struct adapter *adapter)
5371 /* With RSS set up what to auto clear */
5372 if (adapter->msix_mem) {
5373 u32 mask = (adapter->que_mask | adapter->link_mask);
5374 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
5375 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
5376 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
5377 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5380 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
5383 E1000_WRITE_FLUSH(&adapter->hw);
5389 igb_disable_intr(struct adapter *adapter)
5391 if (adapter->msix_mem) {
5392 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
5393 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
5395 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
5396 E1000_WRITE_FLUSH(&adapter->hw);
5401 * Bit of a misnomer, what this really means is
5402 * to enable OS management of the system... aka
5403 * to disable special hardware management features
5406 igb_init_manageability(struct adapter *adapter)
5408 if (adapter->has_manage) {
5409 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
5410 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5412 /* disable hardware interception of ARP */
5413 manc &= ~(E1000_MANC_ARP_EN);
5415 /* enable receiving management packets to the host */
5416 manc |= E1000_MANC_EN_MNG2HOST;
5417 manc2h |= 1 << 5; /* Mng Port 623 */
5418 manc2h |= 1 << 6; /* Mng Port 664 */
5419 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
5420 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5425 * Give control back to hardware management
5426 * controller if there is one.
5429 igb_release_manageability(struct adapter *adapter)
5431 if (adapter->has_manage) {
5432 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5434 /* re-enable hardware interception of ARP */
5435 manc |= E1000_MANC_ARP_EN;
5436 manc &= ~E1000_MANC_EN_MNG2HOST;
5438 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5443 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
5444 * For ASF and Pass Through versions of f/w this means that
5445 * the driver is loaded.
5449 igb_get_hw_control(struct adapter *adapter)
5453 if (adapter->vf_ifp)
5456 /* Let firmware know the driver has taken over */
5457 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5458 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5459 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
5463 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
5464 * For ASF and Pass Through versions of f/w this means that the
5465 * driver is no longer loaded.
5469 igb_release_hw_control(struct adapter *adapter)
5473 if (adapter->vf_ifp)
5476 /* Let firmware taken over control of h/w */
5477 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5478 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5479 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
5483 igb_is_valid_ether_addr(uint8_t *addr)
5485 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
5487 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
5496 * Enable PCI Wake On Lan capability
5499 igb_enable_wakeup(device_t dev)
5504 /* First find the capabilities pointer*/
5505 cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
5506 /* Read the PM Capabilities */
5507 id = pci_read_config(dev, cap, 1);
5508 if (id != PCIY_PMG) /* Something wrong */
5510 /* OK, we have the power capabilities, so
5511 now get the status register */
5512 cap += PCIR_POWER_STATUS;
5513 status = pci_read_config(dev, cap, 2);
5514 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5515 pci_write_config(dev, cap, status, 2);
5520 igb_led_func(void *arg, int onoff)
5522 struct adapter *adapter = arg;
5524 IGB_CORE_LOCK(adapter);
5526 e1000_setup_led(&adapter->hw);
5527 e1000_led_on(&adapter->hw);
5529 e1000_led_off(&adapter->hw);
5530 e1000_cleanup_led(&adapter->hw);
5532 IGB_CORE_UNLOCK(adapter);
5536 igb_get_vf_counter(if_t ifp, ift_counter cnt)
5538 struct adapter *adapter;
5539 struct e1000_vf_stats *stats;
5540 #ifndef IGB_LEGACY_TX
5541 struct tx_ring *txr;
5545 adapter = if_getsoftc(ifp);
5546 stats = (struct e1000_vf_stats *)adapter->stats;
5549 case IFCOUNTER_IPACKETS:
5550 return (stats->gprc);
5551 case IFCOUNTER_OPACKETS:
5552 return (stats->gptc);
5553 case IFCOUNTER_IBYTES:
5554 return (stats->gorc);
5555 case IFCOUNTER_OBYTES:
5556 return (stats->gotc);
5557 case IFCOUNTER_IMCASTS:
5558 return (stats->mprc);
5559 case IFCOUNTER_IERRORS:
5560 return (adapter->dropped_pkts);
5561 case IFCOUNTER_OERRORS:
5562 return (adapter->watchdog_events);
5563 #ifndef IGB_LEGACY_TX
5564 case IFCOUNTER_OQDROPS:
5566 txr = adapter->tx_rings;
5567 for (int i = 0; i < adapter->num_queues; i++, txr++)
5568 rv += txr->br->br_drops;
5572 return (if_get_counter_default(ifp, cnt));
5577 igb_get_counter(if_t ifp, ift_counter cnt)
5579 struct adapter *adapter;
5580 struct e1000_hw_stats *stats;
5581 #ifndef IGB_LEGACY_TX
5582 struct tx_ring *txr;
5586 adapter = if_getsoftc(ifp);
5587 if (adapter->vf_ifp)
5588 return (igb_get_vf_counter(ifp, cnt));
5590 stats = (struct e1000_hw_stats *)adapter->stats;
5593 case IFCOUNTER_IPACKETS:
5594 return (stats->gprc);
5595 case IFCOUNTER_OPACKETS:
5596 return (stats->gptc);
5597 case IFCOUNTER_IBYTES:
5598 return (stats->gorc);
5599 case IFCOUNTER_OBYTES:
5600 return (stats->gotc);
5601 case IFCOUNTER_IMCASTS:
5602 return (stats->mprc);
5603 case IFCOUNTER_OMCASTS:
5604 return (stats->mptc);
5605 case IFCOUNTER_IERRORS:
5606 return (adapter->dropped_pkts + stats->rxerrc +
5607 stats->crcerrs + stats->algnerrc +
5608 stats->ruc + stats->roc + stats->cexterr);
5609 case IFCOUNTER_OERRORS:
5610 return (stats->ecol + stats->latecol +
5611 adapter->watchdog_events);
5612 case IFCOUNTER_COLLISIONS:
5613 return (stats->colc);
5614 case IFCOUNTER_IQDROPS:
5615 return (stats->mpc);
5616 #ifndef IGB_LEGACY_TX
5617 case IFCOUNTER_OQDROPS:
5619 txr = adapter->tx_rings;
5620 for (int i = 0; i < adapter->num_queues; i++, txr++)
5621 rv += txr->br->br_drops;
5625 return (if_get_counter_default(ifp, cnt));
5629 /**********************************************************************
5631 * Update the board statistics counters.
5633 **********************************************************************/
5635 igb_update_stats_counters(struct adapter *adapter)
5637 struct e1000_hw *hw = &adapter->hw;
5638 struct e1000_hw_stats *stats;
5641 ** The virtual function adapter has only a
5642 ** small controlled set of stats, do only
5643 ** those and return.
5645 if (adapter->vf_ifp) {
5646 igb_update_vf_stats_counters(adapter);
5650 stats = (struct e1000_hw_stats *)adapter->stats;
5652 if (adapter->hw.phy.media_type == e1000_media_type_copper ||
5653 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
5655 E1000_READ_REG(hw,E1000_SYMERRS);
5656 stats->sec += E1000_READ_REG(hw, E1000_SEC);
5659 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5660 stats->mpc += E1000_READ_REG(hw, E1000_MPC);
5661 stats->scc += E1000_READ_REG(hw, E1000_SCC);
5662 stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
5664 stats->mcc += E1000_READ_REG(hw, E1000_MCC);
5665 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
5666 stats->colc += E1000_READ_REG(hw, E1000_COLC);
5667 stats->dc += E1000_READ_REG(hw, E1000_DC);
5668 stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
5669 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5670 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5672 ** For watchdog management we need to know if we have been
5673 ** paused during the last interval, so capture that here.
5675 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5676 stats->xoffrxc += adapter->pause_frames;
5677 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5678 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5679 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
5680 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
5681 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
5682 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
5683 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5684 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5685 stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
5686 stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
5687 stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
5688 stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
5690 /* For the 64-bit byte counters the low dword must be read first. */
5691 /* Both registers clear on the read of the high dword */
5693 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
5694 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
5695 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
5696 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
5698 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
5699 stats->ruc += E1000_READ_REG(hw, E1000_RUC);
5700 stats->rfc += E1000_READ_REG(hw, E1000_RFC);
5701 stats->roc += E1000_READ_REG(hw, E1000_ROC);
5702 stats->rjc += E1000_READ_REG(hw, E1000_RJC);
5704 stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5705 stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC);
5706 stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5708 stats->tor += E1000_READ_REG(hw, E1000_TORL) +
5709 ((u64)E1000_READ_REG(hw, E1000_TORH) << 32);
5710 stats->tot += E1000_READ_REG(hw, E1000_TOTL) +
5711 ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32);
5713 stats->tpr += E1000_READ_REG(hw, E1000_TPR);
5714 stats->tpt += E1000_READ_REG(hw, E1000_TPT);
5715 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5716 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5717 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5718 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5719 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5720 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5721 stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
5722 stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
5724 /* Interrupt Counts */
5726 stats->iac += E1000_READ_REG(hw, E1000_IAC);
5727 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5728 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5729 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5730 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5731 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5732 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5733 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5734 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5736 /* Host to Card Statistics */
5738 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
5739 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
5740 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
5741 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
5742 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
5743 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
5744 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
5745 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
5746 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
5747 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
5748 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
5749 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
5750 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
5751 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
5753 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5754 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5755 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5756 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
5757 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5758 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5760 /* Driver specific counters */
5761 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
5762 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
5763 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
5764 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
5765 adapter->packet_buf_alloc_tx =
5766 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
5767 adapter->packet_buf_alloc_rx =
5768 (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
5772 /**********************************************************************
5774 * Initialize the VF board statistics counters.
5776 **********************************************************************/
5778 igb_vf_init_stats(struct adapter *adapter)
5780 struct e1000_hw *hw = &adapter->hw;
5781 struct e1000_vf_stats *stats;
5783 stats = (struct e1000_vf_stats *)adapter->stats;
5786 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
5787 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
5788 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
5789 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
5790 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
5793 /**********************************************************************
5795 * Update the VF board statistics counters.
5797 **********************************************************************/
5799 igb_update_vf_stats_counters(struct adapter *adapter)
5801 struct e1000_hw *hw = &adapter->hw;
5802 struct e1000_vf_stats *stats;
5804 if (adapter->link_speed == 0)
5807 stats = (struct e1000_vf_stats *)adapter->stats;
5809 UPDATE_VF_REG(E1000_VFGPRC,
5810 stats->last_gprc, stats->gprc);
5811 UPDATE_VF_REG(E1000_VFGORC,
5812 stats->last_gorc, stats->gorc);
5813 UPDATE_VF_REG(E1000_VFGPTC,
5814 stats->last_gptc, stats->gptc);
5815 UPDATE_VF_REG(E1000_VFGOTC,
5816 stats->last_gotc, stats->gotc);
5817 UPDATE_VF_REG(E1000_VFMPRC,
5818 stats->last_mprc, stats->mprc);
5821 /* Export a single 32-bit register via a read-only sysctl. */
5823 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5825 struct adapter *adapter;
5828 adapter = oidp->oid_arg1;
5829 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5830 return (sysctl_handle_int(oidp, &val, 0, req));
5834 ** Tuneable interrupt rate handler
5837 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5839 struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1);
5841 u32 reg, usec, rate;
5843 reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
5844 usec = ((reg & 0x7FFC) >> 2);
5846 rate = 1000000 / usec;
5849 error = sysctl_handle_int(oidp, &rate, 0, req);
5850 if (error || !req->newptr)
5856 * Add sysctl variables, one per statistic, to the system.
5859 igb_add_hw_stats(struct adapter *adapter)
5861 device_t dev = adapter->dev;
5863 struct tx_ring *txr = adapter->tx_rings;
5864 struct rx_ring *rxr = adapter->rx_rings;
5866 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5867 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5868 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5869 struct e1000_hw_stats *stats = adapter->stats;
5871 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
5872 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
5874 #define QUEUE_NAME_LEN 32
5875 char namebuf[QUEUE_NAME_LEN];
5877 /* Driver Statistics */
5878 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5879 CTLFLAG_RD, &adapter->dropped_pkts,
5880 "Driver dropped packets");
5881 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5882 CTLFLAG_RD, &adapter->link_irq,
5883 "Link MSIX IRQ Handled");
5884 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
5885 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5886 "Defragmenting mbuf chain failed");
5887 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5888 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5889 "Driver tx dma failure in xmit");
5890 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5891 CTLFLAG_RD, &adapter->rx_overruns,
5893 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5894 CTLFLAG_RD, &adapter->watchdog_events,
5895 "Watchdog timeouts");
5897 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
5898 CTLFLAG_RD, &adapter->device_control,
5899 "Device Control Register");
5900 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
5901 CTLFLAG_RD, &adapter->rx_control,
5902 "Receiver Control Register");
5903 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
5904 CTLFLAG_RD, &adapter->int_mask,
5906 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
5907 CTLFLAG_RD, &adapter->eint_mask,
5908 "Extended Interrupt Mask");
5909 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
5910 CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
5911 "Transmit Buffer Packet Allocation");
5912 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
5913 CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
5914 "Receive Buffer Packet Allocation");
5915 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5916 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5917 "Flow Control High Watermark");
5918 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5919 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5920 "Flow Control Low Watermark");
5922 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5923 struct lro_ctrl *lro = &rxr->lro;
5925 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5926 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5927 CTLFLAG_RD, NULL, "Queue Name");
5928 queue_list = SYSCTL_CHILDREN(queue_node);
5930 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5931 CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i],
5932 sizeof(&adapter->queues[i]),
5933 igb_sysctl_interrupt_rate_handler,
5934 "IU", "Interrupt Rate");
5936 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5937 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me),
5938 igb_sysctl_reg_handler, "IU",
5939 "Transmit Descriptor Head");
5940 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5941 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me),
5942 igb_sysctl_reg_handler, "IU",
5943 "Transmit Descriptor Tail");
5944 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5945 CTLFLAG_RD, &txr->no_desc_avail,
5946 "Queue Descriptors Unavailable");
5947 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5948 CTLFLAG_RD, &txr->total_packets,
5949 "Queue Packets Transmitted");
5951 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5952 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
5953 igb_sysctl_reg_handler, "IU",
5954 "Receive Descriptor Head");
5955 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5956 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
5957 igb_sysctl_reg_handler, "IU",
5958 "Receive Descriptor Tail");
5959 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5960 CTLFLAG_RD, &rxr->rx_packets,
5961 "Queue Packets Received");
5962 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5963 CTLFLAG_RD, &rxr->rx_bytes,
5964 "Queue Bytes Received");
5965 SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_queued",
5966 CTLFLAG_RD, &lro->lro_queued, 0,
5968 SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_flushed",
5969 CTLFLAG_RD, &lro->lro_flushed, 0,
5973 /* MAC stats get their own sub node */
5975 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5976 CTLFLAG_RD, NULL, "MAC Statistics");
5977 stat_list = SYSCTL_CHILDREN(stat_node);
5980 ** VF adapter has a very limited set of stats
5981 ** since its not managing the metal, so to speak.
5983 if (adapter->vf_ifp) {
5984 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5985 CTLFLAG_RD, &stats->gprc,
5986 "Good Packets Received");
5987 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5988 CTLFLAG_RD, &stats->gptc,
5989 "Good Packets Transmitted");
5990 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5991 CTLFLAG_RD, &stats->gorc,
5992 "Good Octets Received");
5993 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5994 CTLFLAG_RD, &stats->gotc,
5995 "Good Octets Transmitted");
5996 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5997 CTLFLAG_RD, &stats->mprc,
5998 "Multicast Packets Received");
6002 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
6003 CTLFLAG_RD, &stats->ecol,
6004 "Excessive collisions");
6005 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
6006 CTLFLAG_RD, &stats->scc,
6007 "Single collisions");
6008 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
6009 CTLFLAG_RD, &stats->mcc,
6010 "Multiple collisions");
6011 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
6012 CTLFLAG_RD, &stats->latecol,
6014 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
6015 CTLFLAG_RD, &stats->colc,
6017 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
6018 CTLFLAG_RD, &stats->symerrs,
6020 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
6021 CTLFLAG_RD, &stats->sec,
6023 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
6024 CTLFLAG_RD, &stats->dc,
6026 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
6027 CTLFLAG_RD, &stats->mpc,
6029 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors",
6030 CTLFLAG_RD, &stats->rlec,
6031 "Receive Length Errors");
6032 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
6033 CTLFLAG_RD, &stats->rnbc,
6034 "Receive No Buffers");
6035 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
6036 CTLFLAG_RD, &stats->ruc,
6037 "Receive Undersize");
6038 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
6039 CTLFLAG_RD, &stats->rfc,
6040 "Fragmented Packets Received");
6041 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
6042 CTLFLAG_RD, &stats->roc,
6043 "Oversized Packets Received");
6044 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
6045 CTLFLAG_RD, &stats->rjc,
6047 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
6048 CTLFLAG_RD, &stats->rxerrc,
6050 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
6051 CTLFLAG_RD, &stats->crcerrs,
6053 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
6054 CTLFLAG_RD, &stats->algnerrc,
6055 "Alignment Errors");
6056 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs",
6057 CTLFLAG_RD, &stats->tncrs,
6058 "Transmit with No CRS");
6059 /* On 82575 these are collision counts */
6060 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
6061 CTLFLAG_RD, &stats->cexterr,
6062 "Collision/Carrier extension errors");
6063 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
6064 CTLFLAG_RD, &stats->xonrxc,
6066 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
6067 CTLFLAG_RD, &stats->xontxc,
6069 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
6070 CTLFLAG_RD, &stats->xoffrxc,
6072 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
6073 CTLFLAG_RD, &stats->xofftxc,
6074 "XOFF Transmitted");
6075 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd",
6076 CTLFLAG_RD, &stats->fcruc,
6077 "Unsupported Flow Control Received");
6078 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd",
6079 CTLFLAG_RD, &stats->mgprc,
6080 "Management Packets Received");
6081 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop",
6082 CTLFLAG_RD, &stats->mgpdc,
6083 "Management Packets Dropped");
6084 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd",
6085 CTLFLAG_RD, &stats->mgptc,
6086 "Management Packets Transmitted");
6087 /* Packet Reception Stats */
6088 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
6089 CTLFLAG_RD, &stats->tpr,
6090 "Total Packets Received");
6091 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
6092 CTLFLAG_RD, &stats->gprc,
6093 "Good Packets Received");
6094 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
6095 CTLFLAG_RD, &stats->bprc,
6096 "Broadcast Packets Received");
6097 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
6098 CTLFLAG_RD, &stats->mprc,
6099 "Multicast Packets Received");
6100 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
6101 CTLFLAG_RD, &stats->prc64,
6102 "64 byte frames received");
6103 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
6104 CTLFLAG_RD, &stats->prc127,
6105 "65-127 byte frames received");
6106 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
6107 CTLFLAG_RD, &stats->prc255,
6108 "128-255 byte frames received");
6109 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
6110 CTLFLAG_RD, &stats->prc511,
6111 "256-511 byte frames received");
6112 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
6113 CTLFLAG_RD, &stats->prc1023,
6114 "512-1023 byte frames received");
6115 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
6116 CTLFLAG_RD, &stats->prc1522,
6117 "1023-1522 byte frames received");
6118 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
6119 CTLFLAG_RD, &stats->gorc,
6120 "Good Octets Received");
6121 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd",
6122 CTLFLAG_RD, &stats->tor,
6123 "Total Octets Received");
6125 /* Packet Transmission Stats */
6126 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
6127 CTLFLAG_RD, &stats->gotc,
6128 "Good Octets Transmitted");
6129 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd",
6130 CTLFLAG_RD, &stats->tot,
6131 "Total Octets Transmitted");
6132 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
6133 CTLFLAG_RD, &stats->tpt,
6134 "Total Packets Transmitted");
6135 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
6136 CTLFLAG_RD, &stats->gptc,
6137 "Good Packets Transmitted");
6138 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
6139 CTLFLAG_RD, &stats->bptc,
6140 "Broadcast Packets Transmitted");
6141 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
6142 CTLFLAG_RD, &stats->mptc,
6143 "Multicast Packets Transmitted");
6144 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
6145 CTLFLAG_RD, &stats->ptc64,
6146 "64 byte frames transmitted");
6147 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
6148 CTLFLAG_RD, &stats->ptc127,
6149 "65-127 byte frames transmitted");
6150 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
6151 CTLFLAG_RD, &stats->ptc255,
6152 "128-255 byte frames transmitted");
6153 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
6154 CTLFLAG_RD, &stats->ptc511,
6155 "256-511 byte frames transmitted");
6156 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
6157 CTLFLAG_RD, &stats->ptc1023,
6158 "512-1023 byte frames transmitted");
6159 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
6160 CTLFLAG_RD, &stats->ptc1522,
6161 "1024-1522 byte frames transmitted");
6162 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
6163 CTLFLAG_RD, &stats->tsctc,
6164 "TSO Contexts Transmitted");
6165 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
6166 CTLFLAG_RD, &stats->tsctfc,
6167 "TSO Contexts Failed");
6170 /* Interrupt Stats */
6172 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
6173 CTLFLAG_RD, NULL, "Interrupt Statistics");
6174 int_list = SYSCTL_CHILDREN(int_node);
6176 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
6177 CTLFLAG_RD, &stats->iac,
6178 "Interrupt Assertion Count");
6180 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
6181 CTLFLAG_RD, &stats->icrxptc,
6182 "Interrupt Cause Rx Pkt Timer Expire Count");
6184 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
6185 CTLFLAG_RD, &stats->icrxatc,
6186 "Interrupt Cause Rx Abs Timer Expire Count");
6188 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
6189 CTLFLAG_RD, &stats->ictxptc,
6190 "Interrupt Cause Tx Pkt Timer Expire Count");
6192 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
6193 CTLFLAG_RD, &stats->ictxatc,
6194 "Interrupt Cause Tx Abs Timer Expire Count");
6196 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
6197 CTLFLAG_RD, &stats->ictxqec,
6198 "Interrupt Cause Tx Queue Empty Count");
6200 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
6201 CTLFLAG_RD, &stats->ictxqmtc,
6202 "Interrupt Cause Tx Queue Min Thresh Count");
6204 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
6205 CTLFLAG_RD, &stats->icrxdmtc,
6206 "Interrupt Cause Rx Desc Min Thresh Count");
6208 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
6209 CTLFLAG_RD, &stats->icrxoc,
6210 "Interrupt Cause Receiver Overrun Count");
6212 /* Host to Card Stats */
6214 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
6216 "Host to Card Statistics");
6218 host_list = SYSCTL_CHILDREN(host_node);
6220 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
6221 CTLFLAG_RD, &stats->cbtmpc,
6222 "Circuit Breaker Tx Packet Count");
6224 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
6225 CTLFLAG_RD, &stats->htdpmc,
6226 "Host Transmit Discarded Packets");
6228 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
6229 CTLFLAG_RD, &stats->rpthc,
6230 "Rx Packets To Host");
6232 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
6233 CTLFLAG_RD, &stats->cbrmpc,
6234 "Circuit Breaker Rx Packet Count");
6236 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
6237 CTLFLAG_RD, &stats->cbrdpc,
6238 "Circuit Breaker Rx Dropped Count");
6240 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
6241 CTLFLAG_RD, &stats->hgptc,
6242 "Host Good Packets Tx Count");
6244 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
6245 CTLFLAG_RD, &stats->htcbdpc,
6246 "Host Tx Circuit Breaker Dropped Count");
6248 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
6249 CTLFLAG_RD, &stats->hgorc,
6250 "Host Good Octets Received Count");
6252 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
6253 CTLFLAG_RD, &stats->hgotc,
6254 "Host Good Octets Transmit Count");
6256 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
6257 CTLFLAG_RD, &stats->lenerrs,
6260 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
6261 CTLFLAG_RD, &stats->scvpc,
6262 "SerDes/SGMII Code Violation Pkt Count");
6264 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
6265 CTLFLAG_RD, &stats->hrmpc,
6266 "Header Redirection Missed Packet Count");
6270 /**********************************************************************
6272 * This routine provides a way to dump out the adapter eeprom,
6273 * often a useful debug/service tool. This only dumps the first
6274 * 32 words, stuff that matters is in that extent.
6276 **********************************************************************/
6278 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
6280 struct adapter *adapter;
6285 error = sysctl_handle_int(oidp, &result, 0, req);
6287 if (error || !req->newptr)
6291 * This value will cause a hex dump of the
6292 * first 32 16-bit words of the EEPROM to
6296 adapter = (struct adapter *)arg1;
6297 igb_print_nvm_info(adapter);
6304 igb_print_nvm_info(struct adapter *adapter)
6309 /* Its a bit crude, but it gets the job done */
6310 printf("\nInterface EEPROM Dump:\n");
6311 printf("Offset\n0x0000 ");
6312 for (i = 0, j = 0; i < 32; i++, j++) {
6313 if (j == 8) { /* Make the offset block */
6315 printf("\n0x00%x0 ",row);
6317 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
6318 printf("%04x ", eeprom_data);
6324 igb_set_sysctl_value(struct adapter *adapter, const char *name,
6325 const char *description, int *limit, int value)
6328 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
6329 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
6330 OID_AUTO, name, CTLFLAG_RW, limit, value, description);
6334 ** Set flow control using sysctl:
6335 ** Flow control values:
6342 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
6345 static int input = 3; /* default is full */
6346 struct adapter *adapter = (struct adapter *) arg1;
6348 error = sysctl_handle_int(oidp, &input, 0, req);
6350 if ((error) || (req->newptr == NULL))
6354 case e1000_fc_rx_pause:
6355 case e1000_fc_tx_pause:
6358 adapter->hw.fc.requested_mode = input;
6359 adapter->fc = input;
6366 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
6367 e1000_force_mac_fc(&adapter->hw);
6368 /* XXX TODO: update DROP_EN on each RX queue if appropriate */
6373 ** Manage DMA Coalesce:
6376 ** Legal timer values are:
6377 ** 250,500,1000-10000 in thousands
6380 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
6382 struct adapter *adapter = (struct adapter *) arg1;
6385 error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
6387 if ((error) || (req->newptr == NULL))
6390 switch (adapter->dmac) {
6394 case 1: /* Just enable and use default */
6395 adapter->dmac = 1000;
6409 /* Legal values - allow */
6412 /* Do nothing, illegal value */
6416 /* Reinit the interface */
6422 ** Manage Energy Efficient Ethernet:
6424 ** 0/1 - enabled/disabled
6427 igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
6429 struct adapter *adapter = (struct adapter *) arg1;
6432 value = adapter->hw.dev_spec._82575.eee_disable;
6433 error = sysctl_handle_int(oidp, &value, 0, req);
6434 if (error || req->newptr == NULL)
6436 IGB_CORE_LOCK(adapter);
6437 adapter->hw.dev_spec._82575.eee_disable = (value != 0);
6438 igb_init_locked(adapter);
6439 IGB_CORE_UNLOCK(adapter);