1 /******************************************************************************
3 Copyright (c) 2001-2013, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.5.15";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 #ifdef IXGBE_LEGACY_TX
106 static void ixgbe_start(struct ifnet *);
107 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
108 #else /* ! IXGBE_LEGACY_TX */
109 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
110 static int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
111 static void ixgbe_qflush(struct ifnet *);
112 static void ixgbe_deferred_mq_start(void *, int);
113 #endif /* IXGBE_LEGACY_TX */
114 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
115 static void ixgbe_init(void *);
116 static void ixgbe_init_locked(struct adapter *);
117 static void ixgbe_stop(void *);
118 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
119 static int ixgbe_media_change(struct ifnet *);
120 static void ixgbe_identify_hardware(struct adapter *);
121 static int ixgbe_allocate_pci_resources(struct adapter *);
122 static void ixgbe_get_slot_info(struct ixgbe_hw *);
123 static int ixgbe_allocate_msix(struct adapter *);
124 static int ixgbe_allocate_legacy(struct adapter *);
125 static int ixgbe_allocate_queues(struct adapter *);
126 static int ixgbe_setup_msix(struct adapter *);
127 static void ixgbe_free_pci_resources(struct adapter *);
128 static void ixgbe_local_timer(void *);
129 static int ixgbe_setup_interface(device_t, struct adapter *);
130 static void ixgbe_config_link(struct adapter *);
132 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133 static int ixgbe_setup_transmit_structures(struct adapter *);
134 static void ixgbe_setup_transmit_ring(struct tx_ring *);
135 static void ixgbe_initialize_transmit_units(struct adapter *);
136 static void ixgbe_free_transmit_structures(struct adapter *);
137 static void ixgbe_free_transmit_buffers(struct tx_ring *);
139 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140 static int ixgbe_setup_receive_structures(struct adapter *);
141 static int ixgbe_setup_receive_ring(struct rx_ring *);
142 static void ixgbe_initialize_receive_units(struct adapter *);
143 static void ixgbe_free_receive_structures(struct adapter *);
144 static void ixgbe_free_receive_buffers(struct rx_ring *);
145 static void ixgbe_setup_hw_rsc(struct rx_ring *);
147 static void ixgbe_enable_intr(struct adapter *);
148 static void ixgbe_disable_intr(struct adapter *);
149 static void ixgbe_update_stats_counters(struct adapter *);
150 static void ixgbe_txeof(struct tx_ring *);
151 static bool ixgbe_rxeof(struct ix_queue *);
152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void ixgbe_set_promisc(struct adapter *);
154 static void ixgbe_set_multi(struct adapter *);
155 static void ixgbe_update_link_status(struct adapter *);
156 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
162 struct ixgbe_dma_alloc *, int);
163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static int ixgbe_tx_ctx_setup(struct tx_ring *,
165 struct mbuf *, u32 *, u32 *);
166 static int ixgbe_tso_setup(struct tx_ring *,
167 struct mbuf *, u32 *, u32 *);
168 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
169 static void ixgbe_configure_ivars(struct adapter *);
170 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
172 static void ixgbe_setup_vlan_hw_support(struct adapter *);
173 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
174 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
176 static void ixgbe_add_hw_stats(struct adapter *adapter);
178 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
179 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
182 static void ixgbe_enable_rx_drop(struct adapter *);
183 static void ixgbe_disable_rx_drop(struct adapter *);
185 /* Support for pluggable optic modules */
186 static bool ixgbe_sfp_probe(struct adapter *);
187 static void ixgbe_setup_optics(struct adapter *);
189 /* Legacy (single vector interrupt handler */
190 static void ixgbe_legacy_irq(void *);
192 /* The MSI/X Interrupt handlers */
193 static void ixgbe_msix_que(void *);
194 static void ixgbe_msix_link(void *);
196 /* Deferred interrupt tasklets */
197 static void ixgbe_handle_que(void *, int);
198 static void ixgbe_handle_link(void *, int);
199 static void ixgbe_handle_msf(void *, int);
200 static void ixgbe_handle_mod(void *, int);
203 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
204 static void ixgbe_reinit_fdir(void *, int);
207 /* Missing shared code prototype */
208 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
210 /*********************************************************************
211 * FreeBSD Device Interface Entry Points
212 *********************************************************************/
214 static device_method_t ixgbe_methods[] = {
215 /* Device interface */
216 DEVMETHOD(device_probe, ixgbe_probe),
217 DEVMETHOD(device_attach, ixgbe_attach),
218 DEVMETHOD(device_detach, ixgbe_detach),
219 DEVMETHOD(device_shutdown, ixgbe_shutdown),
223 static driver_t ixgbe_driver = {
224 "ix", ixgbe_methods, sizeof(struct adapter),
227 devclass_t ixgbe_devclass;
228 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
230 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
231 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
234 ** TUNEABLE PARAMETERS:
237 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD, 0,
238 "IXGBE driver parameters");
241 ** AIM: Adaptive Interrupt Moderation
242 ** which means that the interrupt rate
243 ** is varied over time based on the
244 ** traffic for that interrupt vector
246 static int ixgbe_enable_aim = TRUE;
247 TUNABLE_INT("hw.ix.enable_aim", &ixgbe_enable_aim);
248 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RW, &ixgbe_enable_aim, 0,
249 "Enable adaptive interrupt moderation");
251 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
252 TUNABLE_INT("hw.ix.max_interrupt_rate", &ixgbe_max_interrupt_rate);
253 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
254 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
256 /* How many packets rxeof tries to clean at a time */
257 static int ixgbe_rx_process_limit = 256;
258 TUNABLE_INT("hw.ix.rx_process_limit", &ixgbe_rx_process_limit);
259 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
260 &ixgbe_rx_process_limit, 0,
261 "Maximum number of received packets to process at a time,"
262 "-1 means unlimited");
264 /* How many packets txeof tries to clean at a time */
265 static int ixgbe_tx_process_limit = 256;
266 TUNABLE_INT("hw.ix.tx_process_limit", &ixgbe_tx_process_limit);
267 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
268 &ixgbe_tx_process_limit, 0,
269 "Maximum number of sent packets to process at a time,"
270 "-1 means unlimited");
273 ** Smart speed setting, default to on
274 ** this only works as a compile option
275 ** right now as its during attach, set
276 ** this to 'ixgbe_smart_speed_off' to
279 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
282 * MSIX should be the default for best performance,
283 * but this allows it to be forced off for testing.
285 static int ixgbe_enable_msix = 1;
286 TUNABLE_INT("hw.ix.enable_msix", &ixgbe_enable_msix);
287 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
288 "Enable MSI-X interrupts");
291 * Number of Queues, can be set to 0,
292 * it then autoconfigures based on the
293 * number of cpus with a max of 8. This
294 * can be overriden manually here.
296 static int ixgbe_num_queues = 0;
297 TUNABLE_INT("hw.ix.num_queues", &ixgbe_num_queues);
298 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
299 "Number of queues to configure, 0 indicates autoconfigure");
302 ** Number of TX descriptors per ring,
303 ** setting higher than RX as this seems
304 ** the better performing choice.
306 static int ixgbe_txd = PERFORM_TXD;
307 TUNABLE_INT("hw.ix.txd", &ixgbe_txd);
308 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
309 "Number of transmit descriptors per queue");
311 /* Number of RX descriptors per ring */
312 static int ixgbe_rxd = PERFORM_RXD;
313 TUNABLE_INT("hw.ix.rxd", &ixgbe_rxd);
314 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
315 "Number of receive descriptors per queue");
318 ** Defining this on will allow the use
319 ** of unsupported SFP+ modules, note that
320 ** doing so you are on your own :)
322 static int allow_unsupported_sfp = FALSE;
323 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp);
327 ** this feature only works with
328 ** IPv4, and only on 82599 and later.
329 ** Also this will cause IP forwarding to
330 ** fail and that can't be controlled by
331 ** the stack as LRO can. For all these
332 ** reasons I've deemed it best to leave
333 ** this off and not bother with a tuneable
334 ** interface, this would need to be compiled
337 static bool ixgbe_rsc_enable = FALSE;
339 /* Keep running tab on them for sanity check */
340 static int ixgbe_total_ports;
344 ** For Flow Director: this is the
345 ** number of TX packets we sample
346 ** for the filter pool, this means
347 ** every 20th packet will be probed.
349 ** This feature can be disabled by
350 ** setting this to 0.
352 static int atr_sample_rate = 20;
354 ** Flow Director actually 'steals'
355 ** part of the packet buffer as its
356 ** filter pool, this variable controls
358 ** 0 = 64K, 1 = 128K, 2 = 256K
360 static int fdir_pballoc = 1;
365 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
366 * be a reference on how to implement netmap support in a driver.
367 * Additional comments are in ixgbe_netmap.h .
369 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
370 * that extend the standard driver.
372 #include <dev/netmap/ixgbe_netmap.h>
373 #endif /* DEV_NETMAP */
375 /*********************************************************************
376 * Device identification routine
378 * ixgbe_probe determines if the driver should be loaded on
379 * adapter based on PCI vendor/device id of the adapter.
381 * return BUS_PROBE_DEFAULT on success, positive on failure
382 *********************************************************************/
385 ixgbe_probe(device_t dev)
387 ixgbe_vendor_info_t *ent;
389 u16 pci_vendor_id = 0;
390 u16 pci_device_id = 0;
391 u16 pci_subvendor_id = 0;
392 u16 pci_subdevice_id = 0;
393 char adapter_name[256];
395 INIT_DEBUGOUT("ixgbe_probe: begin");
397 pci_vendor_id = pci_get_vendor(dev);
398 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
401 pci_device_id = pci_get_device(dev);
402 pci_subvendor_id = pci_get_subvendor(dev);
403 pci_subdevice_id = pci_get_subdevice(dev);
405 ent = ixgbe_vendor_info_array;
406 while (ent->vendor_id != 0) {
407 if ((pci_vendor_id == ent->vendor_id) &&
408 (pci_device_id == ent->device_id) &&
410 ((pci_subvendor_id == ent->subvendor_id) ||
411 (ent->subvendor_id == 0)) &&
413 ((pci_subdevice_id == ent->subdevice_id) ||
414 (ent->subdevice_id == 0))) {
415 sprintf(adapter_name, "%s, Version - %s",
416 ixgbe_strings[ent->index],
417 ixgbe_driver_version);
418 device_set_desc_copy(dev, adapter_name);
420 return (BUS_PROBE_DEFAULT);
427 /*********************************************************************
428 * Device initialization routine
430 * The attach entry point is called when the driver is being loaded.
431 * This routine identifies the type of hardware, allocates all resources
432 * and initializes the hardware.
434 * return 0 on success, positive on failure
435 *********************************************************************/
438 ixgbe_attach(device_t dev)
440 struct adapter *adapter;
446 INIT_DEBUGOUT("ixgbe_attach: begin");
448 /* Allocate, clear, and link in our adapter structure */
449 adapter = device_get_softc(dev);
450 adapter->dev = adapter->osdep.dev = dev;
454 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
458 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
459 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
460 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
461 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
463 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
464 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
465 OID_AUTO, "enable_aim", CTLFLAG_RW,
466 &ixgbe_enable_aim, 1, "Interrupt Moderation");
469 ** Allow a kind of speed control by forcing the autoneg
470 ** advertised speed list to only a certain value, this
471 ** supports 1G on 82599 devices, and 100Mb on x540.
473 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
474 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
475 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
476 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
478 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
479 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
480 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
481 0, ixgbe_set_thermal_test, "I", "Thermal Test");
483 /* Set up the timer callout */
484 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
486 /* Determine hardware revision */
487 ixgbe_identify_hardware(adapter);
489 /* Do base PCI setup - map BAR0 */
490 if (ixgbe_allocate_pci_resources(adapter)) {
491 device_printf(dev, "Allocation of PCI resources failed\n");
496 /* Do descriptor calc and sanity checks */
497 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
498 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
499 device_printf(dev, "TXD config issue, using default!\n");
500 adapter->num_tx_desc = DEFAULT_TXD;
502 adapter->num_tx_desc = ixgbe_txd;
505 ** With many RX rings it is easy to exceed the
506 ** system mbuf allocation. Tuning nmbclusters
507 ** can alleviate this.
509 if (nmbclusters > 0 ) {
511 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
512 if (s > nmbclusters) {
513 device_printf(dev, "RX Descriptors exceed "
514 "system mbuf max, using default instead!\n");
515 ixgbe_rxd = DEFAULT_RXD;
519 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
520 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) {
521 device_printf(dev, "RXD config issue, using default!\n");
522 adapter->num_rx_desc = DEFAULT_RXD;
524 adapter->num_rx_desc = ixgbe_rxd;
526 /* Allocate our TX/RX Queues */
527 if (ixgbe_allocate_queues(adapter)) {
532 /* Allocate multicast array memory. */
533 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
534 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
535 if (adapter->mta == NULL) {
536 device_printf(dev, "Can not allocate multicast setup array\n");
541 /* Initialize the shared code */
542 hw->allow_unsupported_sfp = allow_unsupported_sfp;
543 error = ixgbe_init_shared_code(hw);
544 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
546 ** No optics in this port, set up
547 ** so the timer routine will probe
548 ** for later insertion.
550 adapter->sfp_probe = TRUE;
552 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
553 device_printf(dev,"Unsupported SFP+ module detected!\n");
557 device_printf(dev,"Unable to initialize the shared code\n");
562 /* Make sure we have a good EEPROM before we read from it */
563 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
564 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
569 error = ixgbe_init_hw(hw);
571 case IXGBE_ERR_EEPROM_VERSION:
572 device_printf(dev, "This device is a pre-production adapter/"
573 "LOM. Please be aware there may be issues associated "
574 "with your hardware.\n If you are experiencing problems "
575 "please contact your Intel or hardware representative "
576 "who provided you with this hardware.\n");
578 case IXGBE_ERR_SFP_NOT_SUPPORTED:
579 device_printf(dev,"Unsupported SFP+ Module\n");
582 case IXGBE_ERR_SFP_NOT_PRESENT:
583 device_printf(dev,"No SFP+ Module found\n");
589 /* Detect and set physical type */
590 ixgbe_setup_optics(adapter);
592 if ((adapter->msix > 1) && (ixgbe_enable_msix))
593 error = ixgbe_allocate_msix(adapter);
595 error = ixgbe_allocate_legacy(adapter);
599 /* Setup OS specific network interface */
600 if (ixgbe_setup_interface(dev, adapter) != 0)
603 /* Initialize statistics */
604 ixgbe_update_stats_counters(adapter);
606 /* Register for VLAN events */
607 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
608 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
609 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
610 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
613 ** Check PCIE slot type/speed/width
615 ixgbe_get_slot_info(hw);
617 /* Set an initial default flow control value */
618 adapter->fc = ixgbe_fc_full;
620 /* let hardware know driver is loaded */
621 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
622 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
623 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
625 ixgbe_add_hw_stats(adapter);
628 ixgbe_netmap_attach(adapter);
629 #endif /* DEV_NETMAP */
630 INIT_DEBUGOUT("ixgbe_attach: end");
633 ixgbe_free_transmit_structures(adapter);
634 ixgbe_free_receive_structures(adapter);
636 if (adapter->ifp != NULL)
637 if_free(adapter->ifp);
638 ixgbe_free_pci_resources(adapter);
639 free(adapter->mta, M_DEVBUF);
644 /*********************************************************************
645 * Device removal routine
647 * The detach entry point is called when the driver is being removed.
648 * This routine stops the adapter and deallocates all the resources
649 * that were allocated for driver operation.
651 * return 0 on success, positive on failure
652 *********************************************************************/
655 ixgbe_detach(device_t dev)
657 struct adapter *adapter = device_get_softc(dev);
658 struct ix_queue *que = adapter->queues;
659 struct tx_ring *txr = adapter->tx_rings;
662 INIT_DEBUGOUT("ixgbe_detach: begin");
664 /* Make sure VLANS are not using driver */
665 if (adapter->ifp->if_vlantrunk != NULL) {
666 device_printf(dev,"Vlan in use, detach first\n");
670 IXGBE_CORE_LOCK(adapter);
672 IXGBE_CORE_UNLOCK(adapter);
674 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
676 #ifndef IXGBE_LEGACY_TX
677 taskqueue_drain(que->tq, &txr->txq_task);
679 taskqueue_drain(que->tq, &que->que_task);
680 taskqueue_free(que->tq);
684 /* Drain the Link queue */
686 taskqueue_drain(adapter->tq, &adapter->link_task);
687 taskqueue_drain(adapter->tq, &adapter->mod_task);
688 taskqueue_drain(adapter->tq, &adapter->msf_task);
690 taskqueue_drain(adapter->tq, &adapter->fdir_task);
692 taskqueue_free(adapter->tq);
695 /* let hardware know driver is unloading */
696 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
697 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
698 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
700 /* Unregister VLAN events */
701 if (adapter->vlan_attach != NULL)
702 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
703 if (adapter->vlan_detach != NULL)
704 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
706 ether_ifdetach(adapter->ifp);
707 callout_drain(&adapter->timer);
709 netmap_detach(adapter->ifp);
710 #endif /* DEV_NETMAP */
711 ixgbe_free_pci_resources(adapter);
712 bus_generic_detach(dev);
713 if_free(adapter->ifp);
715 ixgbe_free_transmit_structures(adapter);
716 ixgbe_free_receive_structures(adapter);
717 free(adapter->mta, M_DEVBUF);
719 IXGBE_CORE_LOCK_DESTROY(adapter);
723 /*********************************************************************
725 * Shutdown entry point
727 **********************************************************************/
730 ixgbe_shutdown(device_t dev)
732 struct adapter *adapter = device_get_softc(dev);
733 IXGBE_CORE_LOCK(adapter);
735 IXGBE_CORE_UNLOCK(adapter);
740 #ifdef IXGBE_LEGACY_TX
741 /*********************************************************************
742 * Transmit entry point
744 * ixgbe_start is called by the stack to initiate a transmit.
745 * The driver will remain in this routine as long as there are
746 * packets to transmit and transmit resources are available.
747 * In case resources are not available stack is notified and
748 * the packet is requeued.
749 **********************************************************************/
752 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
755 struct adapter *adapter = txr->adapter;
757 IXGBE_TX_LOCK_ASSERT(txr);
759 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
761 if (!adapter->link_active)
764 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
765 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
768 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
772 if (ixgbe_xmit(txr, &m_head)) {
774 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
777 /* Send a copy of the frame to the BPF listener */
778 ETHER_BPF_MTAP(ifp, m_head);
780 /* Set watchdog on */
781 txr->watchdog_time = ticks;
782 txr->queue_status = IXGBE_QUEUE_WORKING;
789 * Legacy TX start - called by the stack, this
790 * always uses the first tx ring, and should
791 * not be used with multiqueue tx enabled.
794 ixgbe_start(struct ifnet *ifp)
796 struct adapter *adapter = ifp->if_softc;
797 struct tx_ring *txr = adapter->tx_rings;
799 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
801 ixgbe_start_locked(txr, ifp);
802 IXGBE_TX_UNLOCK(txr);
807 #else /* ! IXGBE_LEGACY_TX */
810 ** Multiqueue Transmit driver
814 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
816 struct adapter *adapter = ifp->if_softc;
817 struct ix_queue *que;
821 /* Which queue to use */
822 if ((m->m_flags & M_FLOWID) != 0)
823 i = m->m_pkthdr.flowid % adapter->num_queues;
825 i = curcpu % adapter->num_queues;
827 txr = &adapter->tx_rings[i];
828 que = &adapter->queues[i];
830 err = drbr_enqueue(ifp, txr->br, m);
833 if (IXGBE_TX_TRYLOCK(txr)) {
834 ixgbe_mq_start_locked(ifp, txr);
835 IXGBE_TX_UNLOCK(txr);
837 taskqueue_enqueue(que->tq, &txr->txq_task);
843 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
845 struct adapter *adapter = txr->adapter;
847 int enqueued = 0, err = 0;
849 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
850 adapter->link_active == 0)
853 /* Process the queue */
854 #if __FreeBSD_version < 901504
855 next = drbr_dequeue(ifp, txr->br);
856 while (next != NULL) {
857 if ((err = ixgbe_xmit(txr, &next)) != 0) {
859 err = drbr_enqueue(ifp, txr->br, next);
861 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
862 if ((err = ixgbe_xmit(txr, &next)) != 0) {
864 drbr_advance(ifp, txr->br);
866 drbr_putback(ifp, txr->br, next);
871 #if __FreeBSD_version >= 901504
872 drbr_advance(ifp, txr->br);
875 /* Send a copy of the frame to the BPF listener */
876 ETHER_BPF_MTAP(ifp, next);
877 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
879 #if __FreeBSD_version < 901504
880 next = drbr_dequeue(ifp, txr->br);
885 /* Set watchdog on */
886 txr->queue_status = IXGBE_QUEUE_WORKING;
887 txr->watchdog_time = ticks;
890 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
897 * Called from a taskqueue to drain queued transmit packets.
900 ixgbe_deferred_mq_start(void *arg, int pending)
902 struct tx_ring *txr = arg;
903 struct adapter *adapter = txr->adapter;
904 struct ifnet *ifp = adapter->ifp;
907 if (!drbr_empty(ifp, txr->br))
908 ixgbe_mq_start_locked(ifp, txr);
909 IXGBE_TX_UNLOCK(txr);
913 ** Flush all ring buffers
916 ixgbe_qflush(struct ifnet *ifp)
918 struct adapter *adapter = ifp->if_softc;
919 struct tx_ring *txr = adapter->tx_rings;
922 for (int i = 0; i < adapter->num_queues; i++, txr++) {
924 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
926 IXGBE_TX_UNLOCK(txr);
930 #endif /* IXGBE_LEGACY_TX */
932 /*********************************************************************
935 * ixgbe_ioctl is called when the user wants to configure the
938 * return 0 on success, positive on failure
939 **********************************************************************/
942 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
944 struct adapter *adapter = ifp->if_softc;
945 struct ixgbe_hw *hw = &adapter->hw;
946 struct ifreq *ifr = (struct ifreq *) data;
947 #if defined(INET) || defined(INET6)
948 struct ifaddr *ifa = (struct ifaddr *)data;
949 bool avoid_reset = FALSE;
957 if (ifa->ifa_addr->sa_family == AF_INET)
961 if (ifa->ifa_addr->sa_family == AF_INET6)
964 #if defined(INET) || defined(INET6)
966 ** Calling init results in link renegotiation,
967 ** so we avoid doing it when possible.
970 ifp->if_flags |= IFF_UP;
971 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
973 if (!(ifp->if_flags & IFF_NOARP))
974 arp_ifinit(ifp, ifa);
976 error = ether_ioctl(ifp, command, data);
980 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
981 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
984 IXGBE_CORE_LOCK(adapter);
985 ifp->if_mtu = ifr->ifr_mtu;
986 adapter->max_frame_size =
987 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
988 ixgbe_init_locked(adapter);
989 IXGBE_CORE_UNLOCK(adapter);
993 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
994 IXGBE_CORE_LOCK(adapter);
995 if (ifp->if_flags & IFF_UP) {
996 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
997 if ((ifp->if_flags ^ adapter->if_flags) &
998 (IFF_PROMISC | IFF_ALLMULTI)) {
999 ixgbe_set_promisc(adapter);
1002 ixgbe_init_locked(adapter);
1004 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1005 ixgbe_stop(adapter);
1006 adapter->if_flags = ifp->if_flags;
1007 IXGBE_CORE_UNLOCK(adapter);
1011 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1012 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1013 IXGBE_CORE_LOCK(adapter);
1014 ixgbe_disable_intr(adapter);
1015 ixgbe_set_multi(adapter);
1016 ixgbe_enable_intr(adapter);
1017 IXGBE_CORE_UNLOCK(adapter);
1022 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1023 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1027 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1028 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1029 if (mask & IFCAP_HWCSUM)
1030 ifp->if_capenable ^= IFCAP_HWCSUM;
1031 if (mask & IFCAP_TSO4)
1032 ifp->if_capenable ^= IFCAP_TSO4;
1033 if (mask & IFCAP_TSO6)
1034 ifp->if_capenable ^= IFCAP_TSO6;
1035 if (mask & IFCAP_LRO)
1036 ifp->if_capenable ^= IFCAP_LRO;
1037 if (mask & IFCAP_VLAN_HWTAGGING)
1038 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1039 if (mask & IFCAP_VLAN_HWFILTER)
1040 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1041 if (mask & IFCAP_VLAN_HWTSO)
1042 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1043 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1044 IXGBE_CORE_LOCK(adapter);
1045 ixgbe_init_locked(adapter);
1046 IXGBE_CORE_UNLOCK(adapter);
1048 VLAN_CAPABILITIES(ifp);
1053 struct ixgbe_i2c_req i2c;
1054 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1055 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1058 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1062 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1063 i2c.dev_addr, i2c.data);
1064 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1068 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1069 error = ether_ioctl(ifp, command, data);
1076 /*********************************************************************
1079 * This routine is used in two ways. It is used by the stack as
1080 * init entry point in network interface structure. It is also used
1081 * by the driver as a hw/sw initialization routine to get to a
1084 * return 0 on success, positive on failure
1085 **********************************************************************/
1086 #define IXGBE_MHADD_MFS_SHIFT 16
1089 ixgbe_init_locked(struct adapter *adapter)
1091 struct ifnet *ifp = adapter->ifp;
1092 device_t dev = adapter->dev;
1093 struct ixgbe_hw *hw = &adapter->hw;
1094 u32 k, txdctl, mhadd, gpie;
1097 mtx_assert(&adapter->core_mtx, MA_OWNED);
1098 INIT_DEBUGOUT("ixgbe_init_locked: begin");
1099 hw->adapter_stopped = FALSE;
1100 ixgbe_stop_adapter(hw);
1101 callout_stop(&adapter->timer);
1103 /* reprogram the RAR[0] in case user changed it. */
1104 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1106 /* Get the latest mac address, User can use a LAA */
1107 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1108 IXGBE_ETH_LENGTH_OF_ADDRESS);
1109 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1110 hw->addr_ctrl.rar_used_count = 1;
1112 /* Set the various hardware offload abilities */
1113 ifp->if_hwassist = 0;
1114 if (ifp->if_capenable & IFCAP_TSO)
1115 ifp->if_hwassist |= CSUM_TSO;
1116 if (ifp->if_capenable & IFCAP_TXCSUM) {
1117 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1118 #if __FreeBSD_version >= 800000
1119 if (hw->mac.type != ixgbe_mac_82598EB)
1120 ifp->if_hwassist |= CSUM_SCTP;
1124 /* Prepare transmit descriptors and buffers */
1125 if (ixgbe_setup_transmit_structures(adapter)) {
1126 device_printf(dev,"Could not setup transmit structures\n");
1127 ixgbe_stop(adapter);
1132 ixgbe_initialize_transmit_units(adapter);
1134 /* Setup Multicast table */
1135 ixgbe_set_multi(adapter);
1138 ** Determine the correct mbuf pool
1139 ** for doing jumbo frames
1141 if (adapter->max_frame_size <= 2048)
1142 adapter->rx_mbuf_sz = MCLBYTES;
1143 else if (adapter->max_frame_size <= 4096)
1144 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1145 else if (adapter->max_frame_size <= 9216)
1146 adapter->rx_mbuf_sz = MJUM9BYTES;
1148 adapter->rx_mbuf_sz = MJUM16BYTES;
1150 /* Prepare receive descriptors and buffers */
1151 if (ixgbe_setup_receive_structures(adapter)) {
1152 device_printf(dev,"Could not setup receive structures\n");
1153 ixgbe_stop(adapter);
1157 /* Configure RX settings */
1158 ixgbe_initialize_receive_units(adapter);
1160 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1162 /* Enable Fan Failure Interrupt */
1163 gpie |= IXGBE_SDP1_GPIEN;
1165 /* Add for Module detection */
1166 if (hw->mac.type == ixgbe_mac_82599EB)
1167 gpie |= IXGBE_SDP2_GPIEN;
1169 /* Thermal Failure Detection */
1170 if (hw->mac.type == ixgbe_mac_X540)
1171 gpie |= IXGBE_SDP0_GPIEN;
1173 if (adapter->msix > 1) {
1174 /* Enable Enhanced MSIX mode */
1175 gpie |= IXGBE_GPIE_MSIX_MODE;
1176 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1179 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1182 if (ifp->if_mtu > ETHERMTU) {
1183 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1184 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1185 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1186 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1189 /* Now enable all the queues */
1191 for (int i = 0; i < adapter->num_queues; i++) {
1192 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1193 txdctl |= IXGBE_TXDCTL_ENABLE;
1194 /* Set WTHRESH to 8, burst writeback */
1195 txdctl |= (8 << 16);
1197 * When the internal queue falls below PTHRESH (32),
1198 * start prefetching as long as there are at least
1199 * HTHRESH (1) buffers ready. The values are taken
1200 * from the Intel linux driver 3.8.21.
1201 * Prefetching enables tx line rate even with 1 queue.
1203 txdctl |= (32 << 0) | (1 << 8);
1204 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1207 for (int i = 0; i < adapter->num_queues; i++) {
1208 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1209 if (hw->mac.type == ixgbe_mac_82598EB) {
1215 rxdctl &= ~0x3FFFFF;
1218 rxdctl |= IXGBE_RXDCTL_ENABLE;
1219 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1220 for (k = 0; k < 10; k++) {
1221 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1222 IXGBE_RXDCTL_ENABLE)
1230 * In netmap mode, we must preserve the buffers made
1231 * available to userspace before the if_init()
1232 * (this is true by default on the TX side, because
1233 * init makes all buffers available to userspace).
1235 * netmap_reset() and the device specific routines
1236 * (e.g. ixgbe_setup_receive_rings()) map these
1237 * buffers at the end of the NIC ring, so here we
1238 * must set the RDT (tail) register to make sure
1239 * they are not overwritten.
1241 * In this driver the NIC ring starts at RDH = 0,
1242 * RDT points to the last slot available for reception (?),
1243 * so RDT = num_rx_desc - 1 means the whole ring is available.
1245 if (ifp->if_capenable & IFCAP_NETMAP) {
1246 struct netmap_adapter *na = NA(adapter->ifp);
1247 struct netmap_kring *kring = &na->rx_rings[i];
1248 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1250 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1252 #endif /* DEV_NETMAP */
1253 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1256 /* Enable Receive engine */
1257 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1258 if (hw->mac.type == ixgbe_mac_82598EB)
1259 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1260 rxctrl |= IXGBE_RXCTRL_RXEN;
1261 ixgbe_enable_rx_dma(hw, rxctrl);
1263 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1265 /* Set up MSI/X routing */
1266 if (ixgbe_enable_msix) {
1267 ixgbe_configure_ivars(adapter);
1268 /* Set up auto-mask */
1269 if (hw->mac.type == ixgbe_mac_82598EB)
1270 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1272 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1273 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1275 } else { /* Simple settings for Legacy/MSI */
1276 ixgbe_set_ivar(adapter, 0, 0, 0);
1277 ixgbe_set_ivar(adapter, 0, 0, 1);
1278 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1282 /* Init Flow director */
1283 if (hw->mac.type != ixgbe_mac_82598EB) {
1284 u32 hdrm = 32 << fdir_pballoc;
1286 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1287 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1292 ** Check on any SFP devices that
1293 ** need to be kick-started
1295 if (hw->phy.type == ixgbe_phy_none) {
1296 int err = hw->phy.ops.identify(hw);
1297 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1299 "Unsupported SFP+ module type was detected.\n");
1304 /* Set moderation on the Link interrupt */
1305 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1307 /* Config/Enable Link */
1308 ixgbe_config_link(adapter);
1310 /* Hardware Packet Buffer & Flow Control setup */
1312 u32 rxpb, frame, size, tmp;
1314 frame = adapter->max_frame_size;
1316 /* Calculate High Water */
1317 if (hw->mac.type == ixgbe_mac_X540)
1318 tmp = IXGBE_DV_X540(frame, frame);
1320 tmp = IXGBE_DV(frame, frame);
1321 size = IXGBE_BT2KB(tmp);
1322 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1323 hw->fc.high_water[0] = rxpb - size;
1325 /* Now calculate Low Water */
1326 if (hw->mac.type == ixgbe_mac_X540)
1327 tmp = IXGBE_LOW_DV_X540(frame);
1329 tmp = IXGBE_LOW_DV(frame);
1330 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1332 hw->fc.requested_mode = adapter->fc;
1333 hw->fc.pause_time = IXGBE_FC_PAUSE;
1334 hw->fc.send_xon = TRUE;
1336 /* Initialize the FC settings */
1339 /* Set up VLAN support and filter */
1340 ixgbe_setup_vlan_hw_support(adapter);
1342 /* And now turn on interrupts */
1343 ixgbe_enable_intr(adapter);
1345 /* Now inform the stack we're ready */
1346 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1352 ixgbe_init(void *arg)
1354 struct adapter *adapter = arg;
1356 IXGBE_CORE_LOCK(adapter);
1357 ixgbe_init_locked(adapter);
1358 IXGBE_CORE_UNLOCK(adapter);
1365 ** MSIX Interrupt Handlers and Tasklets
1370 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1372 struct ixgbe_hw *hw = &adapter->hw;
1373 u64 queue = (u64)(1 << vector);
1376 if (hw->mac.type == ixgbe_mac_82598EB) {
1377 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1380 mask = (queue & 0xFFFFFFFF);
1382 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1383 mask = (queue >> 32);
1385 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1390 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1392 struct ixgbe_hw *hw = &adapter->hw;
1393 u64 queue = (u64)(1 << vector);
1396 if (hw->mac.type == ixgbe_mac_82598EB) {
1397 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1398 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1400 mask = (queue & 0xFFFFFFFF);
1402 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1403 mask = (queue >> 32);
1405 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1410 ixgbe_handle_que(void *context, int pending)
1412 struct ix_queue *que = context;
1413 struct adapter *adapter = que->adapter;
1414 struct tx_ring *txr = que->txr;
1415 struct ifnet *ifp = adapter->ifp;
1418 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1419 more = ixgbe_rxeof(que);
1422 #ifndef IXGBE_LEGACY_TX
1423 if (!drbr_empty(ifp, txr->br))
1424 ixgbe_mq_start_locked(ifp, txr);
1426 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1427 ixgbe_start_locked(txr, ifp);
1429 IXGBE_TX_UNLOCK(txr);
1432 /* Reenable this interrupt */
1433 if (que->res != NULL)
1434 ixgbe_enable_queue(adapter, que->msix);
1436 ixgbe_enable_intr(adapter);
1441 /*********************************************************************
1443 * Legacy Interrupt Service routine
1445 **********************************************************************/
1448 ixgbe_legacy_irq(void *arg)
1450 struct ix_queue *que = arg;
1451 struct adapter *adapter = que->adapter;
1452 struct ixgbe_hw *hw = &adapter->hw;
1453 struct ifnet *ifp = adapter->ifp;
1454 struct tx_ring *txr = adapter->tx_rings;
1459 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1462 if (reg_eicr == 0) {
1463 ixgbe_enable_intr(adapter);
1467 more = ixgbe_rxeof(que);
1471 #ifdef IXGBE_LEGACY_TX
1472 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1473 ixgbe_start_locked(txr, ifp);
1475 if (!drbr_empty(ifp, txr->br))
1476 ixgbe_mq_start_locked(ifp, txr);
1478 IXGBE_TX_UNLOCK(txr);
1480 /* Check for fan failure */
1481 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1482 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1483 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1484 "REPLACE IMMEDIATELY!!\n");
1485 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1488 /* Link status change */
1489 if (reg_eicr & IXGBE_EICR_LSC)
1490 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1493 taskqueue_enqueue(que->tq, &que->que_task);
1495 ixgbe_enable_intr(adapter);
1500 /*********************************************************************
1502 * MSIX Queue Interrupt Service routine
1504 **********************************************************************/
1506 ixgbe_msix_que(void *arg)
1508 struct ix_queue *que = arg;
1509 struct adapter *adapter = que->adapter;
1510 struct ifnet *ifp = adapter->ifp;
1511 struct tx_ring *txr = que->txr;
1512 struct rx_ring *rxr = que->rxr;
1516 /* Protect against spurious interrupts */
1517 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1520 ixgbe_disable_queue(adapter, que->msix);
1523 more = ixgbe_rxeof(que);
1527 #ifdef IXGBE_LEGACY_TX
1528 if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1529 ixgbe_start_locked(txr, ifp);
1531 if (!drbr_empty(ifp, txr->br))
1532 ixgbe_mq_start_locked(ifp, txr);
1534 IXGBE_TX_UNLOCK(txr);
1538 if (ixgbe_enable_aim == FALSE)
1541 ** Do Adaptive Interrupt Moderation:
1542 ** - Write out last calculated setting
1543 ** - Calculate based on average size over
1544 ** the last interval.
1546 if (que->eitr_setting)
1547 IXGBE_WRITE_REG(&adapter->hw,
1548 IXGBE_EITR(que->msix), que->eitr_setting);
1550 que->eitr_setting = 0;
1552 /* Idle, do nothing */
1553 if ((txr->bytes == 0) && (rxr->bytes == 0))
1556 if ((txr->bytes) && (txr->packets))
1557 newitr = txr->bytes/txr->packets;
1558 if ((rxr->bytes) && (rxr->packets))
1559 newitr = max(newitr,
1560 (rxr->bytes / rxr->packets));
1561 newitr += 24; /* account for hardware frame, crc */
1563 /* set an upper boundary */
1564 newitr = min(newitr, 3000);
1566 /* Be nice to the mid range */
1567 if ((newitr > 300) && (newitr < 1200))
1568 newitr = (newitr / 3);
1570 newitr = (newitr / 2);
1572 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1573 newitr |= newitr << 16;
1575 newitr |= IXGBE_EITR_CNT_WDIS;
1577 /* save for next interrupt */
1578 que->eitr_setting = newitr;
1588 taskqueue_enqueue(que->tq, &que->que_task);
1590 ixgbe_enable_queue(adapter, que->msix);
1596 ixgbe_msix_link(void *arg)
1598 struct adapter *adapter = arg;
1599 struct ixgbe_hw *hw = &adapter->hw;
1602 ++adapter->link_irq;
1604 /* First get the cause */
1605 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1606 /* Be sure the queue bits are not cleared */
1607 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1608 /* Clear interrupt with write */
1609 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1611 /* Link status change */
1612 if (reg_eicr & IXGBE_EICR_LSC)
1613 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1615 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1617 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1618 /* This is probably overkill :) */
1619 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1621 /* Disable the interrupt */
1622 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1623 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1626 if (reg_eicr & IXGBE_EICR_ECC) {
1627 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1628 "Please Reboot!!\n");
1629 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1632 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1633 /* Clear the interrupt */
1634 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1635 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1636 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1637 /* Clear the interrupt */
1638 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1639 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1643 /* Check for fan failure */
1644 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1645 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1646 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1647 "REPLACE IMMEDIATELY!!\n");
1648 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1651 /* Check for over temp condition */
1652 if ((hw->mac.type == ixgbe_mac_X540) &&
1653 (reg_eicr & IXGBE_EICR_TS)) {
1654 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1655 "PHY IS SHUT DOWN!!\n");
1656 device_printf(adapter->dev, "System shutdown required\n");
1657 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1660 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1664 /*********************************************************************
1666 * Media Ioctl callback
1668 * This routine is called whenever the user queries the status of
1669 * the interface using ifconfig.
1671 **********************************************************************/
1673 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1675 struct adapter *adapter = ifp->if_softc;
1677 INIT_DEBUGOUT("ixgbe_media_status: begin");
1678 IXGBE_CORE_LOCK(adapter);
1679 ixgbe_update_link_status(adapter);
1681 ifmr->ifm_status = IFM_AVALID;
1682 ifmr->ifm_active = IFM_ETHER;
1684 if (!adapter->link_active) {
1685 IXGBE_CORE_UNLOCK(adapter);
1689 ifmr->ifm_status |= IFM_ACTIVE;
1691 switch (adapter->link_speed) {
1692 case IXGBE_LINK_SPEED_100_FULL:
1693 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1695 case IXGBE_LINK_SPEED_1GB_FULL:
1696 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1698 case IXGBE_LINK_SPEED_10GB_FULL:
1699 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1703 IXGBE_CORE_UNLOCK(adapter);
1708 /*********************************************************************
1710 * Media Ioctl callback
1712 * This routine is called when the user changes speed/duplex using
1713 * media/mediopt option with ifconfig.
1715 **********************************************************************/
1717 ixgbe_media_change(struct ifnet * ifp)
1719 struct adapter *adapter = ifp->if_softc;
1720 struct ifmedia *ifm = &adapter->media;
1722 INIT_DEBUGOUT("ixgbe_media_change: begin");
1724 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1727 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1729 adapter->hw.phy.autoneg_advertised =
1730 IXGBE_LINK_SPEED_100_FULL |
1731 IXGBE_LINK_SPEED_1GB_FULL |
1732 IXGBE_LINK_SPEED_10GB_FULL;
1735 device_printf(adapter->dev, "Only auto media type\n");
1742 /*********************************************************************
1744 * This routine maps the mbufs to tx descriptors, allowing the
1745 * TX engine to transmit the packets.
1746 * - return 0 on success, positive on failure
1748 **********************************************************************/
1751 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1753 struct adapter *adapter = txr->adapter;
1754 u32 olinfo_status = 0, cmd_type_len;
1755 int i, j, error, nsegs;
1758 struct mbuf *m_head;
1759 bus_dma_segment_t segs[adapter->num_segs];
1761 struct ixgbe_tx_buf *txbuf;
1762 union ixgbe_adv_tx_desc *txd = NULL;
1766 /* Basic descriptor defines */
1767 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1768 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1770 if (m_head->m_flags & M_VLANTAG)
1771 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1774 * Important to capture the first descriptor
1775 * used because it will contain the index of
1776 * the one we tell the hardware to report back
1778 first = txr->next_avail_desc;
1779 txbuf = &txr->tx_buffers[first];
1783 * Map the packet for DMA.
1786 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1787 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1789 if (__predict_false(error)) {
1794 /* Try it again? - one try */
1795 if (remap == TRUE) {
1797 m = m_defrag(*m_headp, M_NOWAIT);
1799 adapter->mbuf_defrag_failed++;
1809 txr->no_tx_dma_setup++;
1812 txr->no_tx_dma_setup++;
1819 /* Make certain there are enough descriptors */
1820 if (nsegs > txr->tx_avail - 2) {
1821 txr->no_desc_avail++;
1822 bus_dmamap_unload(txr->txtag, map);
1828 ** Set up the appropriate offload context
1829 ** this will consume the first descriptor
1831 error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1832 if (__predict_false(error)) {
1833 if (error == ENOBUFS)
1839 /* Do the flow director magic */
1840 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1842 if (txr->atr_count >= atr_sample_rate) {
1843 ixgbe_atr(txr, m_head);
1849 i = txr->next_avail_desc;
1850 for (j = 0; j < nsegs; j++) {
1854 txbuf = &txr->tx_buffers[i];
1855 txd = &txr->tx_base[i];
1856 seglen = segs[j].ds_len;
1857 segaddr = htole64(segs[j].ds_addr);
1859 txd->read.buffer_addr = segaddr;
1860 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1861 cmd_type_len |seglen);
1862 txd->read.olinfo_status = htole32(olinfo_status);
1864 if (++i == txr->num_desc)
1868 txd->read.cmd_type_len |=
1869 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1870 txr->tx_avail -= nsegs;
1871 txr->next_avail_desc = i;
1873 txbuf->m_head = m_head;
1875 ** Here we swap the map so the last descriptor,
1876 ** which gets the completion interrupt has the
1877 ** real map, and the first descriptor gets the
1878 ** unused map from this descriptor.
1880 txr->tx_buffers[first].map = txbuf->map;
1882 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1884 /* Set the EOP descriptor that will be marked done */
1885 txbuf = &txr->tx_buffers[first];
1888 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1889 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1891 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1892 * hardware that this frame is available to transmit.
1894 ++txr->total_packets;
1895 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1902 ixgbe_set_promisc(struct adapter *adapter)
1905 struct ifnet *ifp = adapter->ifp;
1908 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1909 reg_rctl &= (~IXGBE_FCTRL_UPE);
1910 if (ifp->if_flags & IFF_ALLMULTI)
1911 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1913 struct ifmultiaddr *ifma;
1914 #if __FreeBSD_version < 800000
1917 if_maddr_rlock(ifp);
1919 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1920 if (ifma->ifma_addr->sa_family != AF_LINK)
1922 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1926 #if __FreeBSD_version < 800000
1927 IF_ADDR_UNLOCK(ifp);
1929 if_maddr_runlock(ifp);
1932 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1933 reg_rctl &= (~IXGBE_FCTRL_MPE);
1934 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1936 if (ifp->if_flags & IFF_PROMISC) {
1937 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1938 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1939 } else if (ifp->if_flags & IFF_ALLMULTI) {
1940 reg_rctl |= IXGBE_FCTRL_MPE;
1941 reg_rctl &= ~IXGBE_FCTRL_UPE;
1942 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1948 /*********************************************************************
1951 * This routine is called whenever multicast address list is updated.
1953 **********************************************************************/
1954 #define IXGBE_RAR_ENTRIES 16
1957 ixgbe_set_multi(struct adapter *adapter)
1962 struct ifmultiaddr *ifma;
1964 struct ifnet *ifp = adapter->ifp;
1966 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1969 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1970 MAX_NUM_MULTICAST_ADDRESSES);
1972 #if __FreeBSD_version < 800000
1975 if_maddr_rlock(ifp);
1977 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1978 if (ifma->ifma_addr->sa_family != AF_LINK)
1980 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1982 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1983 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1984 IXGBE_ETH_LENGTH_OF_ADDRESS);
1987 #if __FreeBSD_version < 800000
1988 IF_ADDR_UNLOCK(ifp);
1990 if_maddr_runlock(ifp);
1993 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1994 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1995 if (ifp->if_flags & IFF_PROMISC)
1996 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1997 else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1998 ifp->if_flags & IFF_ALLMULTI) {
1999 fctrl |= IXGBE_FCTRL_MPE;
2000 fctrl &= ~IXGBE_FCTRL_UPE;
2002 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2004 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
2006 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
2008 ixgbe_update_mc_addr_list(&adapter->hw,
2009 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
2016 * This is an iterator function now needed by the multicast
2017 * shared code. It simply feeds the shared code routine the
2018 * addresses in the array of ixgbe_set_multi() one by one.
2021 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2023 u8 *addr = *update_ptr;
2027 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2028 *update_ptr = newptr;
2033 /*********************************************************************
2036 * This routine checks for link status,updates statistics,
2037 * and runs the watchdog check.
2039 **********************************************************************/
2042 ixgbe_local_timer(void *arg)
2044 struct adapter *adapter = arg;
2045 device_t dev = adapter->dev;
2046 struct ix_queue *que = adapter->queues;
2047 struct tx_ring *txr = adapter->tx_rings;
2048 int hung = 0, paused = 0;
2050 mtx_assert(&adapter->core_mtx, MA_OWNED);
2052 /* Check for pluggable optics */
2053 if (adapter->sfp_probe)
2054 if (!ixgbe_sfp_probe(adapter))
2055 goto out; /* Nothing to do */
2057 ixgbe_update_link_status(adapter);
2058 ixgbe_update_stats_counters(adapter);
2061 * If the interface has been paused
2062 * then don't do the watchdog check
2064 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2068 ** Check the TX queues status
2069 ** - watchdog only if all queues show hung
2071 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2072 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2075 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2076 taskqueue_enqueue(que->tq, &txr->txq_task);
2078 /* Only truely watchdog if all queues show hung */
2079 if (hung == adapter->num_queues)
2083 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2087 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2088 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2089 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2090 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2091 device_printf(dev,"TX(%d) desc avail = %d,"
2092 "Next TX to Clean = %d\n",
2093 txr->me, txr->tx_avail, txr->next_to_clean);
2094 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2095 adapter->watchdog_events++;
2096 ixgbe_init_locked(adapter);
2100 ** Note: this routine updates the OS on the link state
2101 ** the real check of the hardware only happens with
2102 ** a link interrupt.
2105 ixgbe_update_link_status(struct adapter *adapter)
2107 struct ifnet *ifp = adapter->ifp;
2108 device_t dev = adapter->dev;
2111 if (adapter->link_up){
2112 if (adapter->link_active == FALSE) {
2114 device_printf(dev,"Link is up %d Gbps %s \n",
2115 ((adapter->link_speed == 128)? 10:1),
2117 adapter->link_active = TRUE;
2118 /* Update any Flow Control changes */
2119 ixgbe_fc_enable(&adapter->hw);
2120 if_link_state_change(ifp, LINK_STATE_UP);
2122 } else { /* Link down */
2123 if (adapter->link_active == TRUE) {
2125 device_printf(dev,"Link is Down\n");
2126 if_link_state_change(ifp, LINK_STATE_DOWN);
2127 adapter->link_active = FALSE;
2135 /*********************************************************************
2137 * This routine disables all traffic on the adapter by issuing a
2138 * global reset on the MAC and deallocates TX/RX buffers.
2140 **********************************************************************/
2143 ixgbe_stop(void *arg)
2146 struct adapter *adapter = arg;
2147 struct ixgbe_hw *hw = &adapter->hw;
2150 mtx_assert(&adapter->core_mtx, MA_OWNED);
2152 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2153 ixgbe_disable_intr(adapter);
2154 callout_stop(&adapter->timer);
2156 /* Let the stack know...*/
2157 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2160 hw->adapter_stopped = FALSE;
2161 ixgbe_stop_adapter(hw);
2162 if (hw->mac.type == ixgbe_mac_82599EB)
2163 ixgbe_stop_mac_link_on_d3_82599(hw);
2164 /* Turn off the laser - noop with no optics */
2165 ixgbe_disable_tx_laser(hw);
2167 /* Update the stack */
2168 adapter->link_up = FALSE;
2169 ixgbe_update_link_status(adapter);
2171 /* reprogram the RAR[0] in case user changed it. */
2172 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2178 /*********************************************************************
2180 * Determine hardware revision.
2182 **********************************************************************/
2184 ixgbe_identify_hardware(struct adapter *adapter)
2186 device_t dev = adapter->dev;
2187 struct ixgbe_hw *hw = &adapter->hw;
2189 /* Save off the information about this board */
2190 hw->vendor_id = pci_get_vendor(dev);
2191 hw->device_id = pci_get_device(dev);
2192 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2193 hw->subsystem_vendor_id =
2194 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2195 hw->subsystem_device_id =
2196 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2198 /* We need this here to set the num_segs below */
2199 ixgbe_set_mac_type(hw);
2201 /* Pick up the 82599 and VF settings */
2202 if (hw->mac.type != ixgbe_mac_82598EB) {
2203 hw->phy.smart_speed = ixgbe_smart_speed;
2204 adapter->num_segs = IXGBE_82599_SCATTER;
2206 adapter->num_segs = IXGBE_82598_SCATTER;
2211 /*********************************************************************
2213 * Determine optic type
2215 **********************************************************************/
2217 ixgbe_setup_optics(struct adapter *adapter)
2219 struct ixgbe_hw *hw = &adapter->hw;
2222 layer = ixgbe_get_supported_physical_layer(hw);
2224 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2225 adapter->optics = IFM_10G_T;
2229 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2230 adapter->optics = IFM_1000_T;
2234 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2235 adapter->optics = IFM_1000_SX;
2239 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2240 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2241 adapter->optics = IFM_10G_LR;
2245 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2246 adapter->optics = IFM_10G_SR;
2250 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2251 adapter->optics = IFM_10G_TWINAX;
2255 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2256 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2257 adapter->optics = IFM_10G_CX4;
2261 /* If we get here just set the default */
2262 adapter->optics = IFM_ETHER | IFM_AUTO;
2266 /*********************************************************************
2268 * Setup the Legacy or MSI Interrupt handler
2270 **********************************************************************/
2272 ixgbe_allocate_legacy(struct adapter *adapter)
2274 device_t dev = adapter->dev;
2275 struct ix_queue *que = adapter->queues;
2276 #ifndef IXGBE_LEGACY_TX
2277 struct tx_ring *txr = adapter->tx_rings;
2282 if (adapter->msix == 1)
2285 /* We allocate a single interrupt resource */
2286 adapter->res = bus_alloc_resource_any(dev,
2287 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2288 if (adapter->res == NULL) {
2289 device_printf(dev, "Unable to allocate bus resource: "
2295 * Try allocating a fast interrupt and the associated deferred
2296 * processing contexts.
2298 #ifndef IXGBE_LEGACY_TX
2299 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2301 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2302 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2303 taskqueue_thread_enqueue, &que->tq);
2304 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2305 device_get_nameunit(adapter->dev));
2307 /* Tasklets for Link, SFP and Multispeed Fiber */
2308 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2309 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2310 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2312 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2314 adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2315 taskqueue_thread_enqueue, &adapter->tq);
2316 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2317 device_get_nameunit(adapter->dev));
2319 if ((error = bus_setup_intr(dev, adapter->res,
2320 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2321 que, &adapter->tag)) != 0) {
2322 device_printf(dev, "Failed to register fast interrupt "
2323 "handler: %d\n", error);
2324 taskqueue_free(que->tq);
2325 taskqueue_free(adapter->tq);
2330 /* For simplicity in the handlers */
2331 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2337 /*********************************************************************
2339 * Setup MSIX Interrupt resources and handlers
2341 **********************************************************************/
2343 ixgbe_allocate_msix(struct adapter *adapter)
2345 device_t dev = adapter->dev;
2346 struct ix_queue *que = adapter->queues;
2347 struct tx_ring *txr = adapter->tx_rings;
2348 int error, rid, vector = 0;
2350 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2352 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2353 RF_SHAREABLE | RF_ACTIVE);
2354 if (que->res == NULL) {
2355 device_printf(dev,"Unable to allocate"
2356 " bus resource: que interrupt [%d]\n", vector);
2359 /* Set the handler function */
2360 error = bus_setup_intr(dev, que->res,
2361 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2362 ixgbe_msix_que, que, &que->tag);
2365 device_printf(dev, "Failed to register QUE handler");
2368 #if __FreeBSD_version >= 800504
2369 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2372 adapter->que_mask |= (u64)(1 << que->msix);
2374 ** Bind the msix vector, and thus the
2375 ** ring to the corresponding cpu.
2377 if (adapter->num_queues > 1)
2378 bus_bind_intr(dev, que->res, i);
2380 #ifndef IXGBE_LEGACY_TX
2381 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2383 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2384 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2385 taskqueue_thread_enqueue, &que->tq);
2386 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2387 device_get_nameunit(adapter->dev));
2392 adapter->res = bus_alloc_resource_any(dev,
2393 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2394 if (!adapter->res) {
2395 device_printf(dev,"Unable to allocate"
2396 " bus resource: Link interrupt [%d]\n", rid);
2399 /* Set the link handler function */
2400 error = bus_setup_intr(dev, adapter->res,
2401 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2402 ixgbe_msix_link, adapter, &adapter->tag);
2404 adapter->res = NULL;
2405 device_printf(dev, "Failed to register LINK handler");
2408 #if __FreeBSD_version >= 800504
2409 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2411 adapter->linkvec = vector;
2412 /* Tasklets for Link, SFP and Multispeed Fiber */
2413 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2414 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2415 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2417 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2419 adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2420 taskqueue_thread_enqueue, &adapter->tq);
2421 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2422 device_get_nameunit(adapter->dev));
2428 * Setup Either MSI/X or MSI
2431 ixgbe_setup_msix(struct adapter *adapter)
2433 device_t dev = adapter->dev;
2434 int rid, want, queues, msgs;
2436 /* Override by tuneable */
2437 if (ixgbe_enable_msix == 0)
2440 /* First try MSI/X */
2441 msgs = pci_msix_count(dev);
2444 rid = PCIR_BAR(MSIX_82598_BAR);
2445 adapter->msix_mem = bus_alloc_resource_any(dev,
2446 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2447 if (adapter->msix_mem == NULL) {
2448 rid += 4; /* 82599 maps in higher BAR */
2449 adapter->msix_mem = bus_alloc_resource_any(dev,
2450 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2452 if (adapter->msix_mem == NULL) {
2453 /* May not be enabled */
2454 device_printf(adapter->dev,
2455 "Unable to map MSIX table \n");
2459 /* Figure out a reasonable auto config value */
2460 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2462 if (ixgbe_num_queues != 0)
2463 queues = ixgbe_num_queues;
2464 /* Set max queues to 8 when autoconfiguring */
2465 else if ((ixgbe_num_queues == 0) && (queues > 8))
2468 /* reflect correct sysctl value */
2469 ixgbe_num_queues = queues;
2472 ** Want one vector (RX/TX pair) per queue
2473 ** plus an additional for Link.
2479 device_printf(adapter->dev,
2480 "MSIX Configuration Problem, "
2481 "%d vectors but %d queues wanted!\n",
2485 if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2486 device_printf(adapter->dev,
2487 "Using MSIX interrupts with %d vectors\n", msgs);
2488 adapter->num_queues = queues;
2492 ** If MSIX alloc failed or provided us with
2493 ** less than needed, free and fall through to MSI
2495 pci_release_msi(dev);
2498 if (adapter->msix_mem != NULL) {
2499 bus_release_resource(dev, SYS_RES_MEMORY,
2500 rid, adapter->msix_mem);
2501 adapter->msix_mem = NULL;
2504 if (pci_alloc_msi(dev, &msgs) == 0) {
2505 device_printf(adapter->dev,"Using an MSI interrupt\n");
2508 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2514 ixgbe_allocate_pci_resources(struct adapter *adapter)
2517 device_t dev = adapter->dev;
2520 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2523 if (!(adapter->pci_mem)) {
2524 device_printf(dev,"Unable to allocate bus resource: memory\n");
2528 adapter->osdep.mem_bus_space_tag =
2529 rman_get_bustag(adapter->pci_mem);
2530 adapter->osdep.mem_bus_space_handle =
2531 rman_get_bushandle(adapter->pci_mem);
2532 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2534 /* Legacy defaults */
2535 adapter->num_queues = 1;
2536 adapter->hw.back = &adapter->osdep;
2539 ** Now setup MSI or MSI/X, should
2540 ** return us the number of supported
2541 ** vectors. (Will be 1 for MSI)
2543 adapter->msix = ixgbe_setup_msix(adapter);
2548 ixgbe_free_pci_resources(struct adapter * adapter)
2550 struct ix_queue *que = adapter->queues;
2551 device_t dev = adapter->dev;
2554 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2555 memrid = PCIR_BAR(MSIX_82598_BAR);
2557 memrid = PCIR_BAR(MSIX_82599_BAR);
2560 ** There is a slight possibility of a failure mode
2561 ** in attach that will result in entering this function
2562 ** before interrupt resources have been initialized, and
2563 ** in that case we do not want to execute the loops below
2564 ** We can detect this reliably by the state of the adapter
2567 if (adapter->res == NULL)
2571 ** Release all msix queue resources:
2573 for (int i = 0; i < adapter->num_queues; i++, que++) {
2574 rid = que->msix + 1;
2575 if (que->tag != NULL) {
2576 bus_teardown_intr(dev, que->res, que->tag);
2579 if (que->res != NULL)
2580 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2584 /* Clean the Legacy or Link interrupt last */
2585 if (adapter->linkvec) /* we are doing MSIX */
2586 rid = adapter->linkvec + 1;
2588 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2590 if (adapter->tag != NULL) {
2591 bus_teardown_intr(dev, adapter->res, adapter->tag);
2592 adapter->tag = NULL;
2594 if (adapter->res != NULL)
2595 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2599 pci_release_msi(dev);
2601 if (adapter->msix_mem != NULL)
2602 bus_release_resource(dev, SYS_RES_MEMORY,
2603 memrid, adapter->msix_mem);
2605 if (adapter->pci_mem != NULL)
2606 bus_release_resource(dev, SYS_RES_MEMORY,
2607 PCIR_BAR(0), adapter->pci_mem);
2612 /*********************************************************************
2614 * Setup networking device structure and register an interface.
2616 **********************************************************************/
2618 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2620 struct ixgbe_hw *hw = &adapter->hw;
2623 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2625 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2627 device_printf(dev, "can not allocate ifnet structure\n");
2630 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2631 #if __FreeBSD_version < 1000025
2632 ifp->if_baudrate = 1000000000;
2634 if_initbaudrate(ifp, IF_Gbps(10));
2636 ifp->if_init = ixgbe_init;
2637 ifp->if_softc = adapter;
2638 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2639 ifp->if_ioctl = ixgbe_ioctl;
2640 #ifndef IXGBE_LEGACY_TX
2641 ifp->if_transmit = ixgbe_mq_start;
2642 ifp->if_qflush = ixgbe_qflush;
2644 ifp->if_start = ixgbe_start;
2645 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2646 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2647 IFQ_SET_READY(&ifp->if_snd);
2650 ether_ifattach(ifp, adapter->hw.mac.addr);
2652 adapter->max_frame_size =
2653 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2656 * Tell the upper layer(s) we support long frames.
2658 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2660 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2661 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2662 ifp->if_capabilities |= IFCAP_LRO;
2663 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2667 ifp->if_capenable = ifp->if_capabilities;
2670 ** Don't turn this on by default, if vlans are
2671 ** created on another pseudo device (eg. lagg)
2672 ** then vlan events are not passed thru, breaking
2673 ** operation, but with HW FILTER off it works. If
2674 ** using vlans directly on the ixgbe driver you can
2675 ** enable this and get full hardware tag filtering.
2677 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2680 * Specify the media types supported by this adapter and register
2681 * callbacks to update media and link information
2683 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2684 ixgbe_media_status);
2685 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2686 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2687 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2688 ifmedia_add(&adapter->media,
2689 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2690 ifmedia_add(&adapter->media,
2691 IFM_ETHER | IFM_1000_T, 0, NULL);
2693 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2694 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2700 ixgbe_config_link(struct adapter *adapter)
2702 struct ixgbe_hw *hw = &adapter->hw;
2703 u32 autoneg, err = 0;
2704 bool sfp, negotiate;
2706 sfp = ixgbe_is_sfp(hw);
2709 if (hw->phy.multispeed_fiber) {
2710 hw->mac.ops.setup_sfp(hw);
2711 ixgbe_enable_tx_laser(hw);
2712 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2714 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2716 if (hw->mac.ops.check_link)
2717 err = ixgbe_check_link(hw, &adapter->link_speed,
2718 &adapter->link_up, FALSE);
2721 autoneg = hw->phy.autoneg_advertised;
2722 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2723 err = hw->mac.ops.get_link_capabilities(hw,
2724 &autoneg, &negotiate);
2727 if (hw->mac.ops.setup_link)
2728 err = hw->mac.ops.setup_link(hw,
2729 autoneg, adapter->link_up);
2735 /********************************************************************
2736 * Manage DMA'able memory.
2737 *******************************************************************/
2739 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2743 *(bus_addr_t *) arg = segs->ds_addr;
2748 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2749 struct ixgbe_dma_alloc *dma, int mapflags)
2751 device_t dev = adapter->dev;
2754 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2755 DBA_ALIGN, 0, /* alignment, bounds */
2756 BUS_SPACE_MAXADDR, /* lowaddr */
2757 BUS_SPACE_MAXADDR, /* highaddr */
2758 NULL, NULL, /* filter, filterarg */
2761 size, /* maxsegsize */
2762 BUS_DMA_ALLOCNOW, /* flags */
2763 NULL, /* lockfunc */
2764 NULL, /* lockfuncarg */
2767 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2771 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2772 BUS_DMA_NOWAIT, &dma->dma_map);
2774 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2778 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2782 mapflags | BUS_DMA_NOWAIT);
2784 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2788 dma->dma_size = size;
2791 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2793 bus_dma_tag_destroy(dma->dma_tag);
2795 dma->dma_map = NULL;
2796 dma->dma_tag = NULL;
2801 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2803 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2804 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2805 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2806 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2807 bus_dma_tag_destroy(dma->dma_tag);
2811 /*********************************************************************
2813 * Allocate memory for the transmit and receive rings, and then
2814 * the descriptors associated with each, called only once at attach.
2816 **********************************************************************/
2818 ixgbe_allocate_queues(struct adapter *adapter)
2820 device_t dev = adapter->dev;
2821 struct ix_queue *que;
2822 struct tx_ring *txr;
2823 struct rx_ring *rxr;
2824 int rsize, tsize, error = IXGBE_SUCCESS;
2825 int txconf = 0, rxconf = 0;
2827 /* First allocate the top level queue structs */
2828 if (!(adapter->queues =
2829 (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2830 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2831 device_printf(dev, "Unable to allocate queue memory\n");
2836 /* First allocate the TX ring struct memory */
2837 if (!(adapter->tx_rings =
2838 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2839 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2840 device_printf(dev, "Unable to allocate TX ring memory\n");
2845 /* Next allocate the RX */
2846 if (!(adapter->rx_rings =
2847 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2848 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2849 device_printf(dev, "Unable to allocate RX ring memory\n");
2854 /* For the ring itself */
2855 tsize = roundup2(adapter->num_tx_desc *
2856 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2859 * Now set up the TX queues, txconf is needed to handle the
2860 * possibility that things fail midcourse and we need to
2861 * undo memory gracefully
2863 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2864 /* Set up some basics */
2865 txr = &adapter->tx_rings[i];
2866 txr->adapter = adapter;
2868 txr->num_desc = adapter->num_tx_desc;
2870 /* Initialize the TX side lock */
2871 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2872 device_get_nameunit(dev), txr->me);
2873 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2875 if (ixgbe_dma_malloc(adapter, tsize,
2876 &txr->txdma, BUS_DMA_NOWAIT)) {
2878 "Unable to allocate TX Descriptor memory\n");
2882 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2883 bzero((void *)txr->tx_base, tsize);
2885 /* Now allocate transmit buffers for the ring */
2886 if (ixgbe_allocate_transmit_buffers(txr)) {
2888 "Critical Failure setting up transmit buffers\n");
2892 #ifndef IXGBE_LEGACY_TX
2893 /* Allocate a buf ring */
2894 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2895 M_WAITOK, &txr->tx_mtx);
2896 if (txr->br == NULL) {
2898 "Critical Failure setting up buf ring\n");
2906 * Next the RX queues...
2908 rsize = roundup2(adapter->num_rx_desc *
2909 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2910 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2911 rxr = &adapter->rx_rings[i];
2912 /* Set up some basics */
2913 rxr->adapter = adapter;
2915 rxr->num_desc = adapter->num_rx_desc;
2917 /* Initialize the RX side lock */
2918 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2919 device_get_nameunit(dev), rxr->me);
2920 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2922 if (ixgbe_dma_malloc(adapter, rsize,
2923 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2925 "Unable to allocate RxDescriptor memory\n");
2929 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2930 bzero((void *)rxr->rx_base, rsize);
2932 /* Allocate receive buffers for the ring*/
2933 if (ixgbe_allocate_receive_buffers(rxr)) {
2935 "Critical Failure setting up receive buffers\n");
2942 ** Finally set up the queue holding structs
2944 for (int i = 0; i < adapter->num_queues; i++) {
2945 que = &adapter->queues[i];
2946 que->adapter = adapter;
2947 que->txr = &adapter->tx_rings[i];
2948 que->rxr = &adapter->rx_rings[i];
2954 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2955 ixgbe_dma_free(adapter, &rxr->rxdma);
2957 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2958 ixgbe_dma_free(adapter, &txr->txdma);
2959 free(adapter->rx_rings, M_DEVBUF);
2961 free(adapter->tx_rings, M_DEVBUF);
2963 free(adapter->queues, M_DEVBUF);
2968 /*********************************************************************
2970 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2971 * the information needed to transmit a packet on the wire. This is
2972 * called only once at attach, setup is done every reset.
2974 **********************************************************************/
2976 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2978 struct adapter *adapter = txr->adapter;
2979 device_t dev = adapter->dev;
2980 struct ixgbe_tx_buf *txbuf;
2984 * Setup DMA descriptor areas.
2986 if ((error = bus_dma_tag_create(
2987 bus_get_dma_tag(adapter->dev), /* parent */
2988 1, 0, /* alignment, bounds */
2989 BUS_SPACE_MAXADDR, /* lowaddr */
2990 BUS_SPACE_MAXADDR, /* highaddr */
2991 NULL, NULL, /* filter, filterarg */
2992 IXGBE_TSO_SIZE, /* maxsize */
2993 adapter->num_segs, /* nsegments */
2994 PAGE_SIZE, /* maxsegsize */
2996 NULL, /* lockfunc */
2997 NULL, /* lockfuncarg */
2999 device_printf(dev,"Unable to allocate TX DMA tag\n");
3003 if (!(txr->tx_buffers =
3004 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
3005 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3006 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3011 /* Create the descriptor buffer dma maps */
3012 txbuf = txr->tx_buffers;
3013 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3014 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3016 device_printf(dev, "Unable to create TX DMA map\n");
3023 /* We free all, it handles case where we are in the middle */
3024 ixgbe_free_transmit_structures(adapter);
3028 /*********************************************************************
3030 * Initialize a transmit ring.
3032 **********************************************************************/
3034 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3036 struct adapter *adapter = txr->adapter;
3037 struct ixgbe_tx_buf *txbuf;
3040 struct netmap_adapter *na = NA(adapter->ifp);
3041 struct netmap_slot *slot;
3042 #endif /* DEV_NETMAP */
3044 /* Clear the old ring contents */
3048 * (under lock): if in netmap mode, do some consistency
3049 * checks and set slot to entry 0 of the netmap ring.
3051 slot = netmap_reset(na, NR_TX, txr->me, 0);
3052 #endif /* DEV_NETMAP */
3053 bzero((void *)txr->tx_base,
3054 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3056 txr->next_avail_desc = 0;
3057 txr->next_to_clean = 0;
3059 /* Free any existing tx buffers. */
3060 txbuf = txr->tx_buffers;
3061 for (i = 0; i < txr->num_desc; i++, txbuf++) {
3062 if (txbuf->m_head != NULL) {
3063 bus_dmamap_sync(txr->txtag, txbuf->map,
3064 BUS_DMASYNC_POSTWRITE);
3065 bus_dmamap_unload(txr->txtag, txbuf->map);
3066 m_freem(txbuf->m_head);
3067 txbuf->m_head = NULL;
3071 * In netmap mode, set the map for the packet buffer.
3072 * NOTE: Some drivers (not this one) also need to set
3073 * the physical buffer address in the NIC ring.
3074 * Slots in the netmap ring (indexed by "si") are
3075 * kring->nkr_hwofs positions "ahead" wrt the
3076 * corresponding slot in the NIC ring. In some drivers
3077 * (not here) nkr_hwofs can be negative. Function
3078 * netmap_idx_n2k() handles wraparounds properly.
3081 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3082 netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
3084 #endif /* DEV_NETMAP */
3085 /* Clear the EOP descriptor pointer */
3090 /* Set the rate at which we sample packets */
3091 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3092 txr->atr_sample = atr_sample_rate;
3095 /* Set number of descriptors available */
3096 txr->tx_avail = adapter->num_tx_desc;
3098 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3099 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3100 IXGBE_TX_UNLOCK(txr);
3103 /*********************************************************************
3105 * Initialize all transmit rings.
3107 **********************************************************************/
3109 ixgbe_setup_transmit_structures(struct adapter *adapter)
3111 struct tx_ring *txr = adapter->tx_rings;
3113 for (int i = 0; i < adapter->num_queues; i++, txr++)
3114 ixgbe_setup_transmit_ring(txr);
3119 /*********************************************************************
3121 * Enable transmit unit.
3123 **********************************************************************/
3125 ixgbe_initialize_transmit_units(struct adapter *adapter)
3127 struct tx_ring *txr = adapter->tx_rings;
3128 struct ixgbe_hw *hw = &adapter->hw;
3130 /* Setup the Base and Length of the Tx Descriptor Ring */
3132 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3133 u64 tdba = txr->txdma.dma_paddr;
3136 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3137 (tdba & 0x00000000ffffffffULL));
3138 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3139 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3140 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3142 /* Setup the HW Tx Head and Tail descriptor pointers */
3143 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3144 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3146 /* Setup Transmit Descriptor Cmd Settings */
3147 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3148 txr->queue_status = IXGBE_QUEUE_IDLE;
3150 /* Set the processing limit */
3151 txr->process_limit = ixgbe_tx_process_limit;
3153 /* Disable Head Writeback */
3154 switch (hw->mac.type) {
3155 case ixgbe_mac_82598EB:
3156 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3158 case ixgbe_mac_82599EB:
3159 case ixgbe_mac_X540:
3161 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3164 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3165 switch (hw->mac.type) {
3166 case ixgbe_mac_82598EB:
3167 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3169 case ixgbe_mac_82599EB:
3170 case ixgbe_mac_X540:
3172 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3178 if (hw->mac.type != ixgbe_mac_82598EB) {
3179 u32 dmatxctl, rttdcs;
3180 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3181 dmatxctl |= IXGBE_DMATXCTL_TE;
3182 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3183 /* Disable arbiter to set MTQC */
3184 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3185 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3186 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3187 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3188 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3189 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3195 /*********************************************************************
3197 * Free all transmit rings.
3199 **********************************************************************/
3201 ixgbe_free_transmit_structures(struct adapter *adapter)
3203 struct tx_ring *txr = adapter->tx_rings;
3205 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3207 ixgbe_free_transmit_buffers(txr);
3208 ixgbe_dma_free(adapter, &txr->txdma);
3209 IXGBE_TX_UNLOCK(txr);
3210 IXGBE_TX_LOCK_DESTROY(txr);
3212 free(adapter->tx_rings, M_DEVBUF);
3215 /*********************************************************************
3217 * Free transmit ring related data structures.
3219 **********************************************************************/
3221 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3223 struct adapter *adapter = txr->adapter;
3224 struct ixgbe_tx_buf *tx_buffer;
3227 INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin");
3229 if (txr->tx_buffers == NULL)
3232 tx_buffer = txr->tx_buffers;
3233 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3234 if (tx_buffer->m_head != NULL) {
3235 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3236 BUS_DMASYNC_POSTWRITE);
3237 bus_dmamap_unload(txr->txtag,
3239 m_freem(tx_buffer->m_head);
3240 tx_buffer->m_head = NULL;
3241 if (tx_buffer->map != NULL) {
3242 bus_dmamap_destroy(txr->txtag,
3244 tx_buffer->map = NULL;
3246 } else if (tx_buffer->map != NULL) {
3247 bus_dmamap_unload(txr->txtag,
3249 bus_dmamap_destroy(txr->txtag,
3251 tx_buffer->map = NULL;
3254 #ifdef IXGBE_LEGACY_TX
3255 if (txr->br != NULL)
3256 buf_ring_free(txr->br, M_DEVBUF);
3258 if (txr->tx_buffers != NULL) {
3259 free(txr->tx_buffers, M_DEVBUF);
3260 txr->tx_buffers = NULL;
3262 if (txr->txtag != NULL) {
3263 bus_dma_tag_destroy(txr->txtag);
3269 /*********************************************************************
3271 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3273 **********************************************************************/
3276 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3277 u32 *cmd_type_len, u32 *olinfo_status)
3279 struct ixgbe_adv_tx_context_desc *TXD;
3280 struct ether_vlan_header *eh;
3282 struct ip6_hdr *ip6;
3283 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3284 int ehdrlen, ip_hlen = 0;
3288 int ctxd = txr->next_avail_desc;
3291 /* First check if TSO is to be used */
3292 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3293 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3295 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3298 /* Indicate the whole packet as payload when not doing TSO */
3299 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3301 /* Now ready a context descriptor */
3302 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3305 ** In advanced descriptors the vlan tag must
3306 ** be placed into the context descriptor. Hence
3307 ** we need to make one even if not doing offloads.
3309 if (mp->m_flags & M_VLANTAG) {
3310 vtag = htole16(mp->m_pkthdr.ether_vtag);
3311 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3312 } else if (offload == FALSE) /* ... no offload to do */
3316 * Determine where frame payload starts.
3317 * Jump over vlan headers if already present,
3318 * helpful for QinQ too.
3320 eh = mtod(mp, struct ether_vlan_header *);
3321 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3322 etype = ntohs(eh->evl_proto);
3323 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3325 etype = ntohs(eh->evl_encap_proto);
3326 ehdrlen = ETHER_HDR_LEN;
3329 /* Set the ether header length */
3330 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3334 ip = (struct ip *)(mp->m_data + ehdrlen);
3335 ip_hlen = ip->ip_hl << 2;
3337 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3339 case ETHERTYPE_IPV6:
3340 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3341 ip_hlen = sizeof(struct ip6_hdr);
3342 /* XXX-BZ this will go badly in case of ext hdrs. */
3343 ipproto = ip6->ip6_nxt;
3344 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3351 vlan_macip_lens |= ip_hlen;
3352 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3356 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3357 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3361 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3362 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3365 #if __FreeBSD_version >= 800000
3367 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3368 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3376 if (offload) /* For the TX descriptor setup */
3377 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3379 /* Now copy bits into descriptor */
3380 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3381 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3382 TXD->seqnum_seed = htole32(0);
3383 TXD->mss_l4len_idx = htole32(0);
3385 /* We've consumed the first desc, adjust counters */
3386 if (++ctxd == txr->num_desc)
3388 txr->next_avail_desc = ctxd;
3394 /**********************************************************************
3396 * Setup work for hardware segmentation offload (TSO) on
3397 * adapters using advanced tx descriptors
3399 **********************************************************************/
3401 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3402 u32 *cmd_type_len, u32 *olinfo_status)
3404 struct ixgbe_adv_tx_context_desc *TXD;
3405 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3406 u32 mss_l4len_idx = 0, paylen;
3407 u16 vtag = 0, eh_type;
3408 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3409 struct ether_vlan_header *eh;
3411 struct ip6_hdr *ip6;
3420 * Determine where frame payload starts.
3421 * Jump over vlan headers if already present
3423 eh = mtod(mp, struct ether_vlan_header *);
3424 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3425 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3426 eh_type = eh->evl_proto;
3428 ehdrlen = ETHER_HDR_LEN;
3429 eh_type = eh->evl_encap_proto;
3432 switch (ntohs(eh_type)) {
3434 case ETHERTYPE_IPV6:
3435 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3436 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3437 if (ip6->ip6_nxt != IPPROTO_TCP)
3439 ip_hlen = sizeof(struct ip6_hdr);
3440 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3441 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3442 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3443 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3448 ip = (struct ip *)(mp->m_data + ehdrlen);
3449 if (ip->ip_p != IPPROTO_TCP)
3452 ip_hlen = ip->ip_hl << 2;
3453 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3454 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3455 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3456 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3457 /* Tell transmit desc to also do IPv4 checksum. */
3458 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3462 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3463 __func__, ntohs(eh_type));
3467 ctxd = txr->next_avail_desc;
3468 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3470 tcp_hlen = th->th_off << 2;
3472 /* This is used in the transmit desc in encap */
3473 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3475 /* VLAN MACLEN IPLEN */
3476 if (mp->m_flags & M_VLANTAG) {
3477 vtag = htole16(mp->m_pkthdr.ether_vtag);
3478 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3481 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3482 vlan_macip_lens |= ip_hlen;
3483 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3485 /* ADV DTYPE TUCMD */
3486 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3487 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3488 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3491 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3492 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3493 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3495 TXD->seqnum_seed = htole32(0);
3497 if (++ctxd == txr->num_desc)
3501 txr->next_avail_desc = ctxd;
3502 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3503 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3504 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3511 ** This routine parses packet headers so that Flow
3512 ** Director can make a hashed filter table entry
3513 ** allowing traffic flows to be identified and kept
3514 ** on the same cpu. This would be a performance
3515 ** hit, but we only do it at IXGBE_FDIR_RATE of
3519 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3521 struct adapter *adapter = txr->adapter;
3522 struct ix_queue *que;
3526 struct ether_vlan_header *eh;
3527 union ixgbe_atr_hash_dword input = {.dword = 0};
3528 union ixgbe_atr_hash_dword common = {.dword = 0};
3529 int ehdrlen, ip_hlen;
3532 eh = mtod(mp, struct ether_vlan_header *);
3533 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3534 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3535 etype = eh->evl_proto;
3537 ehdrlen = ETHER_HDR_LEN;
3538 etype = eh->evl_encap_proto;
3541 /* Only handling IPv4 */
3542 if (etype != htons(ETHERTYPE_IP))
3545 ip = (struct ip *)(mp->m_data + ehdrlen);
3546 ip_hlen = ip->ip_hl << 2;
3548 /* check if we're UDP or TCP */
3551 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3552 /* src and dst are inverted */
3553 common.port.dst ^= th->th_sport;
3554 common.port.src ^= th->th_dport;
3555 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3558 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3559 /* src and dst are inverted */
3560 common.port.dst ^= uh->uh_sport;
3561 common.port.src ^= uh->uh_dport;
3562 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3568 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3569 if (mp->m_pkthdr.ether_vtag)
3570 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3572 common.flex_bytes ^= etype;
3573 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3575 que = &adapter->queues[txr->me];
3577 ** This assumes the Rx queue and Tx
3578 ** queue are bound to the same CPU
3580 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3581 input, common, que->msix);
3583 #endif /* IXGBE_FDIR */
3585 /**********************************************************************
3587 * Examine each tx_buffer in the used queue. If the hardware is done
3588 * processing the packet then free associated resources. The
3589 * tx_buffer is put back on the free queue.
3591 **********************************************************************/
3593 ixgbe_txeof(struct tx_ring *txr)
3595 struct adapter *adapter = txr->adapter;
3596 struct ifnet *ifp = adapter->ifp;
3597 u32 work, processed = 0;
3598 u16 limit = txr->process_limit;
3599 struct ixgbe_tx_buf *buf;
3600 union ixgbe_adv_tx_desc *txd;
3602 mtx_assert(&txr->tx_mtx, MA_OWNED);
3605 if (ifp->if_capenable & IFCAP_NETMAP) {
3606 struct netmap_adapter *na = NA(ifp);
3607 struct netmap_kring *kring = &na->tx_rings[txr->me];
3609 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3610 BUS_DMASYNC_POSTREAD);
3612 * In netmap mode, all the work is done in the context
3613 * of the client thread. Interrupt handlers only wake up
3614 * clients, which may be sleeping on individual rings
3615 * or on a global resource for all rings.
3616 * To implement tx interrupt mitigation, we wake up the client
3617 * thread roughly every half ring, even if the NIC interrupts
3618 * more frequently. This is implemented as follows:
3619 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3620 * the slot that should wake up the thread (nkr_num_slots
3621 * means the user thread should not be woken up);
3622 * - the driver ignores tx interrupts unless netmap_mitigate=0
3623 * or the slot has the DD bit set.
3625 * When the driver has separate locks, we need to
3626 * release and re-acquire txlock to avoid deadlocks.
3627 * XXX see if we can find a better way.
3629 if (!netmap_mitigate ||
3630 (kring->nr_kflags < kring->nkr_num_slots &&
3631 txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3632 netmap_tx_irq(ifp, txr->me);
3636 #endif /* DEV_NETMAP */
3638 if (txr->tx_avail == txr->num_desc) {
3639 txr->queue_status = IXGBE_QUEUE_IDLE;
3643 /* Get work starting point */
3644 work = txr->next_to_clean;
3645 buf = &txr->tx_buffers[work];
3646 txd = &txr->tx_base[work];
3647 work -= txr->num_desc; /* The distance to ring end */
3648 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3649 BUS_DMASYNC_POSTREAD);
3652 union ixgbe_adv_tx_desc *eop= buf->eop;
3653 if (eop == NULL) /* No work */
3656 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3657 break; /* I/O not complete */
3661 buf->m_head->m_pkthdr.len;
3662 bus_dmamap_sync(txr->txtag,
3664 BUS_DMASYNC_POSTWRITE);
3665 bus_dmamap_unload(txr->txtag,
3667 m_freem(buf->m_head);
3674 /* We clean the range if multi segment */
3675 while (txd != eop) {
3679 /* wrap the ring? */
3680 if (__predict_false(!work)) {
3681 work -= txr->num_desc;
3682 buf = txr->tx_buffers;
3687 buf->m_head->m_pkthdr.len;
3688 bus_dmamap_sync(txr->txtag,
3690 BUS_DMASYNC_POSTWRITE);
3691 bus_dmamap_unload(txr->txtag,
3693 m_freem(buf->m_head);
3704 txr->watchdog_time = ticks;
3706 /* Try the next packet */
3710 /* reset with a wrap */
3711 if (__predict_false(!work)) {
3712 work -= txr->num_desc;
3713 buf = txr->tx_buffers;
3717 } while (__predict_true(--limit));
3719 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3720 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3722 work += txr->num_desc;
3723 txr->next_to_clean = work;
3726 ** Watchdog calculation, we know there's
3727 ** work outstanding or the first return
3728 ** would have been taken, so none processed
3729 ** for too long indicates a hang.
3731 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3732 txr->queue_status = IXGBE_QUEUE_HUNG;
3734 if (txr->tx_avail == txr->num_desc)
3735 txr->queue_status = IXGBE_QUEUE_IDLE;
3740 /*********************************************************************
3742 * Refresh mbuf buffers for RX descriptor rings
3743 * - now keeps its own state so discards due to resource
3744 * exhaustion are unnecessary, if an mbuf cannot be obtained
3745 * it just returns, keeping its placeholder, thus it can simply
3746 * be recalled to try again.
3748 **********************************************************************/
3750 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3752 struct adapter *adapter = rxr->adapter;
3753 bus_dma_segment_t seg[1];
3754 struct ixgbe_rx_buf *rxbuf;
3756 int i, j, nsegs, error;
3757 bool refreshed = FALSE;
3759 i = j = rxr->next_to_refresh;
3760 /* Control the loop with one beyond */
3761 if (++j == rxr->num_desc)
3764 while (j != limit) {
3765 rxbuf = &rxr->rx_buffers[i];
3766 if (rxbuf->buf == NULL) {
3767 mp = m_getjcl(M_NOWAIT, MT_DATA,
3768 M_PKTHDR, rxr->mbuf_sz);
3771 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3772 m_adj(mp, ETHER_ALIGN);
3776 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3778 /* If we're dealing with an mbuf that was copied rather
3779 * than replaced, there's no need to go through busdma.
3781 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3782 /* Get the memory mapping */
3783 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3784 rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3786 printf("Refresh mbufs: payload dmamap load"
3787 " failure - %d\n", error);
3793 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3794 BUS_DMASYNC_PREREAD);
3795 rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3796 htole64(seg[0].ds_addr);
3798 rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3799 rxbuf->flags &= ~IXGBE_RX_COPY;
3803 /* Next is precalculated */
3805 rxr->next_to_refresh = i;
3806 if (++j == rxr->num_desc)
3810 if (refreshed) /* Update hardware tail index */
3811 IXGBE_WRITE_REG(&adapter->hw,
3812 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3816 /*********************************************************************
3818 * Allocate memory for rx_buffer structures. Since we use one
3819 * rx_buffer per received packet, the maximum number of rx_buffer's
3820 * that we'll need is equal to the number of receive descriptors
3821 * that we've allocated.
3823 **********************************************************************/
3825 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3827 struct adapter *adapter = rxr->adapter;
3828 device_t dev = adapter->dev;
3829 struct ixgbe_rx_buf *rxbuf;
3830 int i, bsize, error;
3832 bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3833 if (!(rxr->rx_buffers =
3834 (struct ixgbe_rx_buf *) malloc(bsize,
3835 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3836 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3841 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3842 1, 0, /* alignment, bounds */
3843 BUS_SPACE_MAXADDR, /* lowaddr */
3844 BUS_SPACE_MAXADDR, /* highaddr */
3845 NULL, NULL, /* filter, filterarg */
3846 MJUM16BYTES, /* maxsize */
3848 MJUM16BYTES, /* maxsegsize */
3850 NULL, /* lockfunc */
3851 NULL, /* lockfuncarg */
3853 device_printf(dev, "Unable to create RX DMA tag\n");
3857 for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3858 rxbuf = &rxr->rx_buffers[i];
3859 error = bus_dmamap_create(rxr->ptag,
3860 BUS_DMA_NOWAIT, &rxbuf->pmap);
3862 device_printf(dev, "Unable to create RX dma map\n");
3870 /* Frees all, but can handle partial completion */
3871 ixgbe_free_receive_structures(adapter);
3876 ** Used to detect a descriptor that has
3877 ** been merged by Hardware RSC.
3880 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3882 return (le32toh(rx->wb.lower.lo_dword.data) &
3883 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3886 /*********************************************************************
3888 * Initialize Hardware RSC (LRO) feature on 82599
3889 * for an RX ring, this is toggled by the LRO capability
3890 * even though it is transparent to the stack.
3892 * NOTE: since this HW feature only works with IPV4 and
3893 * our testing has shown soft LRO to be as effective
3894 * I have decided to disable this by default.
3896 **********************************************************************/
3898 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3900 struct adapter *adapter = rxr->adapter;
3901 struct ixgbe_hw *hw = &adapter->hw;
3902 u32 rscctrl, rdrxctl;
3904 /* If turning LRO/RSC off we need to disable it */
3905 if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3906 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3907 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3911 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3912 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3913 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3914 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3915 #endif /* DEV_NETMAP */
3916 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3917 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3918 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3920 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3921 rscctrl |= IXGBE_RSCCTL_RSCEN;
3923 ** Limit the total number of descriptors that
3924 ** can be combined, so it does not exceed 64K
3926 if (rxr->mbuf_sz == MCLBYTES)
3927 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3928 else if (rxr->mbuf_sz == MJUMPAGESIZE)
3929 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3930 else if (rxr->mbuf_sz == MJUM9BYTES)
3931 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3932 else /* Using 16K cluster */
3933 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3935 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3937 /* Enable TCP header recognition */
3938 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3939 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3940 IXGBE_PSRTYPE_TCPHDR));
3942 /* Disable RSC for ACK packets */
3943 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3944 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3951 ixgbe_free_receive_ring(struct rx_ring *rxr)
3953 struct ixgbe_rx_buf *rxbuf;
3956 for (i = 0; i < rxr->num_desc; i++) {
3957 rxbuf = &rxr->rx_buffers[i];
3958 if (rxbuf->buf != NULL) {
3959 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3960 BUS_DMASYNC_POSTREAD);
3961 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3962 rxbuf->buf->m_flags |= M_PKTHDR;
3963 m_freem(rxbuf->buf);
3971 /*********************************************************************
3973 * Initialize a receive ring and its buffers.
3975 **********************************************************************/
3977 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3979 struct adapter *adapter;
3982 struct ixgbe_rx_buf *rxbuf;
3983 bus_dma_segment_t seg[1];
3984 struct lro_ctrl *lro = &rxr->lro;
3985 int rsize, nsegs, error = 0;
3987 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3988 struct netmap_slot *slot;
3989 #endif /* DEV_NETMAP */
3991 adapter = rxr->adapter;
3995 /* Clear the ring contents */
3998 /* same as in ixgbe_setup_transmit_ring() */
3999 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4000 #endif /* DEV_NETMAP */
4001 rsize = roundup2(adapter->num_rx_desc *
4002 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4003 bzero((void *)rxr->rx_base, rsize);
4004 /* Cache the size */
4005 rxr->mbuf_sz = adapter->rx_mbuf_sz;
4007 /* Free current RX buffer structs and their mbufs */
4008 ixgbe_free_receive_ring(rxr);
4010 /* Now replenish the mbufs */
4011 for (int j = 0; j != rxr->num_desc; ++j) {
4014 rxbuf = &rxr->rx_buffers[j];
4017 * In netmap mode, fill the map and set the buffer
4018 * address in the NIC ring, considering the offset
4019 * between the netmap and NIC rings (see comment in
4020 * ixgbe_setup_transmit_ring() ). No need to allocate
4021 * an mbuf, so end the block with a continue;
4024 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4028 addr = PNMB(na, slot + sj, &paddr);
4029 netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
4030 /* Update descriptor and the cached value */
4031 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4032 rxbuf->addr = htole64(paddr);
4035 #endif /* DEV_NETMAP */
4037 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
4038 M_PKTHDR, adapter->rx_mbuf_sz);
4039 if (rxbuf->buf == NULL) {
4044 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4045 /* Get the memory mapping */
4046 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4047 rxbuf->pmap, mp, seg,
4048 &nsegs, BUS_DMA_NOWAIT);
4051 bus_dmamap_sync(rxr->ptag,
4052 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4053 /* Update the descriptor and the cached value */
4054 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4055 rxbuf->addr = htole64(seg[0].ds_addr);
4059 /* Setup our descriptor indices */
4060 rxr->next_to_check = 0;
4061 rxr->next_to_refresh = 0;
4062 rxr->lro_enabled = FALSE;
4065 rxr->vtag_strip = FALSE;
4067 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4068 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4071 ** Now set up the LRO interface:
4073 if (ixgbe_rsc_enable)
4074 ixgbe_setup_hw_rsc(rxr);
4075 else if (ifp->if_capenable & IFCAP_LRO) {
4076 int err = tcp_lro_init(lro);
4078 device_printf(dev, "LRO Initialization failed!\n");
4081 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4082 rxr->lro_enabled = TRUE;
4083 lro->ifp = adapter->ifp;
4086 IXGBE_RX_UNLOCK(rxr);
4090 ixgbe_free_receive_ring(rxr);
4091 IXGBE_RX_UNLOCK(rxr);
4095 /*********************************************************************
4097 * Initialize all receive rings.
4099 **********************************************************************/
4101 ixgbe_setup_receive_structures(struct adapter *adapter)
4103 struct rx_ring *rxr = adapter->rx_rings;
4106 for (j = 0; j < adapter->num_queues; j++, rxr++)
4107 if (ixgbe_setup_receive_ring(rxr))
4113 * Free RX buffers allocated so far, we will only handle
4114 * the rings that completed, the failing case will have
4115 * cleaned up for itself. 'j' failed, so its the terminus.
4117 for (int i = 0; i < j; ++i) {
4118 rxr = &adapter->rx_rings[i];
4119 ixgbe_free_receive_ring(rxr);
4125 /*********************************************************************
4127 * Setup receive registers and features.
4129 **********************************************************************/
4130 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4132 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4135 ixgbe_initialize_receive_units(struct adapter *adapter)
4137 struct rx_ring *rxr = adapter->rx_rings;
4138 struct ixgbe_hw *hw = &adapter->hw;
4139 struct ifnet *ifp = adapter->ifp;
4140 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
4141 u32 reta, mrqc = 0, hlreg, random[10];
4145 * Make sure receives are disabled while
4146 * setting up the descriptor ring
4148 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4149 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4150 rxctrl & ~IXGBE_RXCTRL_RXEN);
4152 /* Enable broadcasts */
4153 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4154 fctrl |= IXGBE_FCTRL_BAM;
4155 fctrl |= IXGBE_FCTRL_DPF;
4156 fctrl |= IXGBE_FCTRL_PMCF;
4157 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4159 /* Set for Jumbo Frames? */
4160 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4161 if (ifp->if_mtu > ETHERMTU)
4162 hlreg |= IXGBE_HLREG0_JUMBOEN;
4164 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4166 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4167 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4168 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4170 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4171 #endif /* DEV_NETMAP */
4172 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4174 bufsz = (adapter->rx_mbuf_sz +
4175 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4177 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4178 u64 rdba = rxr->rxdma.dma_paddr;
4180 /* Setup the Base and Length of the Rx Descriptor Ring */
4181 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4182 (rdba & 0x00000000ffffffffULL));
4183 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4184 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4185 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4187 /* Set up the SRRCTL register */
4188 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4189 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4190 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4192 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4195 * Set DROP_EN iff we have no flow control and >1 queue.
4196 * Note that srrctl was cleared shortly before during reset,
4197 * so we do not need to clear the bit, but do it just in case
4198 * this code is moved elsewhere.
4200 if (adapter->num_queues > 1 &&
4201 adapter->hw.fc.requested_mode == ixgbe_fc_none) {
4202 srrctl |= IXGBE_SRRCTL_DROP_EN;
4204 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
4207 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4209 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4210 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4211 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4213 /* Set the processing limit */
4214 rxr->process_limit = ixgbe_rx_process_limit;
4217 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4218 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4219 IXGBE_PSRTYPE_UDPHDR |
4220 IXGBE_PSRTYPE_IPV4HDR |
4221 IXGBE_PSRTYPE_IPV6HDR;
4222 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4225 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4228 if (adapter->num_queues > 1) {
4232 /* set up random bits */
4233 arc4rand(&random, sizeof(random), 0);
4235 /* Set up the redirection table */
4236 for (i = 0, j = 0; i < 128; i++, j++) {
4237 if (j == adapter->num_queues) j = 0;
4238 reta = (reta << 8) | (j * 0x11);
4240 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4243 /* Now fill our hash function seeds */
4244 for (int i = 0; i < 10; i++)
4245 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4247 /* Perform hash on these packet types */
4248 mrqc = IXGBE_MRQC_RSSEN
4249 | IXGBE_MRQC_RSS_FIELD_IPV4
4250 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4251 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4252 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4253 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4254 | IXGBE_MRQC_RSS_FIELD_IPV6
4255 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4256 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4257 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4258 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4260 /* RSS and RX IPP Checksum are mutually exclusive */
4261 rxcsum |= IXGBE_RXCSUM_PCSD;
4264 if (ifp->if_capenable & IFCAP_RXCSUM)
4265 rxcsum |= IXGBE_RXCSUM_PCSD;
4267 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4268 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4270 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4275 /*********************************************************************
4277 * Free all receive rings.
4279 **********************************************************************/
4281 ixgbe_free_receive_structures(struct adapter *adapter)
4283 struct rx_ring *rxr = adapter->rx_rings;
4285 INIT_DEBUGOUT("ixgbe_free_receive_structures: begin");
4287 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4288 struct lro_ctrl *lro = &rxr->lro;
4289 ixgbe_free_receive_buffers(rxr);
4290 /* Free LRO memory */
4292 /* Free the ring memory as well */
4293 ixgbe_dma_free(adapter, &rxr->rxdma);
4296 free(adapter->rx_rings, M_DEVBUF);
4300 /*********************************************************************
4302 * Free receive ring data structures
4304 **********************************************************************/
4306 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4308 struct adapter *adapter = rxr->adapter;
4309 struct ixgbe_rx_buf *rxbuf;
4311 INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin");
4313 /* Cleanup any existing buffers */
4314 if (rxr->rx_buffers != NULL) {
4315 for (int i = 0; i < adapter->num_rx_desc; i++) {
4316 rxbuf = &rxr->rx_buffers[i];
4317 if (rxbuf->buf != NULL) {
4318 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4319 BUS_DMASYNC_POSTREAD);
4320 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4321 rxbuf->buf->m_flags |= M_PKTHDR;
4322 m_freem(rxbuf->buf);
4325 if (rxbuf->pmap != NULL) {
4326 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4330 if (rxr->rx_buffers != NULL) {
4331 free(rxr->rx_buffers, M_DEVBUF);
4332 rxr->rx_buffers = NULL;
4336 if (rxr->ptag != NULL) {
4337 bus_dma_tag_destroy(rxr->ptag);
4344 static __inline void
4345 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4349 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4350 * should be computed by hardware. Also it should not have VLAN tag in
4351 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4353 if (rxr->lro_enabled &&
4354 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4355 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4356 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4357 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4358 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4359 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4360 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4361 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4363 * Send to the stack if:
4364 ** - LRO not enabled, or
4365 ** - no LRO resources, or
4366 ** - lro enqueue fails
4368 if (rxr->lro.lro_cnt != 0)
4369 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4372 IXGBE_RX_UNLOCK(rxr);
4373 (*ifp->if_input)(ifp, m);
4377 static __inline void
4378 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4380 struct ixgbe_rx_buf *rbuf;
4382 rbuf = &rxr->rx_buffers[i];
4386 ** With advanced descriptors the writeback
4387 ** clobbers the buffer addrs, so its easier
4388 ** to just free the existing mbufs and take
4389 ** the normal refresh path to get new buffers
4393 if (rbuf->fmp != NULL) {/* Partial chain ? */
4394 rbuf->fmp->m_flags |= M_PKTHDR;
4397 rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */
4398 } else if (rbuf->buf) {
4409 /*********************************************************************
4411 * This routine executes in interrupt context. It replenishes
4412 * the mbufs in the descriptor and sends data which has been
4413 * dma'ed into host memory to upper layer.
4415 * We loop at most count times if count is > 0, or until done if
4418 * Return TRUE for more work, FALSE for all clean.
4419 *********************************************************************/
4421 ixgbe_rxeof(struct ix_queue *que)
4423 struct adapter *adapter = que->adapter;
4424 struct rx_ring *rxr = que->rxr;
4425 struct ifnet *ifp = adapter->ifp;
4426 struct lro_ctrl *lro = &rxr->lro;
4427 struct lro_entry *queued;
4428 int i, nextp, processed = 0;
4430 u16 count = rxr->process_limit;
4431 union ixgbe_adv_rx_desc *cur;
4432 struct ixgbe_rx_buf *rbuf, *nbuf;
4437 /* Same as the txeof routine: wakeup clients on intr. */
4438 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4439 IXGBE_RX_UNLOCK(rxr);
4442 #endif /* DEV_NETMAP */
4444 for (i = rxr->next_to_check; count != 0;) {
4445 struct mbuf *sendmp, *mp;
4451 /* Sync the ring. */
4452 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4453 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4455 cur = &rxr->rx_base[i];
4456 staterr = le32toh(cur->wb.upper.status_error);
4458 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4460 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4467 cur->wb.upper.status_error = 0;
4468 rbuf = &rxr->rx_buffers[i];
4471 len = le16toh(cur->wb.upper.length);
4472 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4473 IXGBE_RXDADV_PKTTYPE_MASK;
4474 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4476 /* Make sure bad packets are discarded */
4477 if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) {
4478 rxr->rx_discarded++;
4479 ixgbe_rx_discard(rxr, i);
4484 ** On 82599 which supports a hardware
4485 ** LRO (called HW RSC), packets need
4486 ** not be fragmented across sequential
4487 ** descriptors, rather the next descriptor
4488 ** is indicated in bits of the descriptor.
4489 ** This also means that we might proceses
4490 ** more than one packet at a time, something
4491 ** that has never been true before, it
4492 ** required eliminating global chain pointers
4493 ** in favor of what we are doing here. -jfv
4497 ** Figure out the next descriptor
4500 if (rxr->hw_rsc == TRUE) {
4501 rsc = ixgbe_rsc_count(cur);
4502 rxr->rsc_num += (rsc - 1);
4504 if (rsc) { /* Get hardware index */
4506 IXGBE_RXDADV_NEXTP_MASK) >>
4507 IXGBE_RXDADV_NEXTP_SHIFT);
4508 } else { /* Just sequential */
4510 if (nextp == adapter->num_rx_desc)
4513 nbuf = &rxr->rx_buffers[nextp];
4517 ** Rather than using the fmp/lmp global pointers
4518 ** we now keep the head of a packet chain in the
4519 ** buffer struct and pass this along from one
4520 ** descriptor to the next, until we get EOP.
4524 ** See if there is a stored head
4525 ** that determines what we are
4528 if (sendmp != NULL) { /* secondary frag */
4529 rbuf->buf = rbuf->fmp = NULL;
4530 mp->m_flags &= ~M_PKTHDR;
4531 sendmp->m_pkthdr.len += mp->m_len;
4534 * Optimize. This might be a small packet,
4535 * maybe just a TCP ACK. Do a fast copy that
4536 * is cache aligned into a new mbuf, and
4537 * leave the old mbuf+cluster for re-use.
4539 if (eop && len <= IXGBE_RX_COPY_LEN) {
4540 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4541 if (sendmp != NULL) {
4543 IXGBE_RX_COPY_ALIGN;
4544 ixgbe_bcopy(mp->m_data,
4545 sendmp->m_data, len);
4546 sendmp->m_len = len;
4548 rbuf->flags |= IXGBE_RX_COPY;
4551 if (sendmp == NULL) {
4552 rbuf->buf = rbuf->fmp = NULL;
4556 /* first desc of a non-ps chain */
4557 sendmp->m_flags |= M_PKTHDR;
4558 sendmp->m_pkthdr.len = mp->m_len;
4562 /* Pass the head pointer on */
4566 mp->m_next = nbuf->buf;
4567 } else { /* Sending this frame */
4568 sendmp->m_pkthdr.rcvif = ifp;
4571 /* capture data for AIM */
4572 rxr->bytes += sendmp->m_pkthdr.len;
4573 rxr->rx_bytes += sendmp->m_pkthdr.len;
4574 /* Process vlan info */
4575 if ((rxr->vtag_strip) &&
4576 (staterr & IXGBE_RXD_STAT_VP))
4577 vtag = le16toh(cur->wb.upper.vlan);
4579 sendmp->m_pkthdr.ether_vtag = vtag;
4580 sendmp->m_flags |= M_VLANTAG;
4582 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4583 ixgbe_rx_checksum(staterr, sendmp, ptype);
4584 #if __FreeBSD_version >= 800000
4585 sendmp->m_pkthdr.flowid = que->msix;
4586 sendmp->m_flags |= M_FLOWID;
4590 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4591 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4593 /* Advance our pointers to the next descriptor. */
4594 if (++i == rxr->num_desc)
4597 /* Now send to the stack or do LRO */
4598 if (sendmp != NULL) {
4599 rxr->next_to_check = i;
4600 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4601 i = rxr->next_to_check;
4604 /* Every 8 descriptors we go to refresh mbufs */
4605 if (processed == 8) {
4606 ixgbe_refresh_mbufs(rxr, i);
4611 /* Refresh any remaining buf structs */
4612 if (ixgbe_rx_unrefreshed(rxr))
4613 ixgbe_refresh_mbufs(rxr, i);
4615 rxr->next_to_check = i;
4618 * Flush any outstanding LRO work
4620 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4621 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4622 tcp_lro_flush(lro, queued);
4625 IXGBE_RX_UNLOCK(rxr);
4628 ** Still have cleaning to do?
4630 if ((staterr & IXGBE_RXD_STAT_DD) != 0)
4637 /*********************************************************************
4639 * Verify that the hardware indicated that the checksum is valid.
4640 * Inform the stack about the status of checksum so that stack
4641 * doesn't spend time verifying the checksum.
4643 *********************************************************************/
4645 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4647 u16 status = (u16) staterr;
4648 u8 errors = (u8) (staterr >> 24);
4651 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4652 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4655 if (status & IXGBE_RXD_STAT_IPCS) {
4656 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4657 /* IP Checksum Good */
4658 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4659 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4662 mp->m_pkthdr.csum_flags = 0;
4664 if (status & IXGBE_RXD_STAT_L4CS) {
4665 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4666 #if __FreeBSD_version >= 800000
4668 type = CSUM_SCTP_VALID;
4670 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4671 mp->m_pkthdr.csum_flags |= type;
4673 mp->m_pkthdr.csum_data = htons(0xffff);
4681 ** This routine is run via an vlan config EVENT,
4682 ** it enables us to use the HW Filter table since
4683 ** we can get the vlan id. This just creates the
4684 ** entry in the soft version of the VFTA, init will
4685 ** repopulate the real table.
4688 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4690 struct adapter *adapter = ifp->if_softc;
4693 if (ifp->if_softc != arg) /* Not our event */
4696 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4699 IXGBE_CORE_LOCK(adapter);
4700 index = (vtag >> 5) & 0x7F;
4702 adapter->shadow_vfta[index] |= (1 << bit);
4703 ++adapter->num_vlans;
4704 ixgbe_setup_vlan_hw_support(adapter);
4705 IXGBE_CORE_UNLOCK(adapter);
4709 ** This routine is run via an vlan
4710 ** unconfig EVENT, remove our entry
4711 ** in the soft vfta.
4714 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4716 struct adapter *adapter = ifp->if_softc;
4719 if (ifp->if_softc != arg)
4722 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4725 IXGBE_CORE_LOCK(adapter);
4726 index = (vtag >> 5) & 0x7F;
4728 adapter->shadow_vfta[index] &= ~(1 << bit);
4729 --adapter->num_vlans;
4730 /* Re-init to load the changes */
4731 ixgbe_setup_vlan_hw_support(adapter);
4732 IXGBE_CORE_UNLOCK(adapter);
4736 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4738 struct ifnet *ifp = adapter->ifp;
4739 struct ixgbe_hw *hw = &adapter->hw;
4740 struct rx_ring *rxr;
4745 ** We get here thru init_locked, meaning
4746 ** a soft reset, this has already cleared
4747 ** the VFTA and other state, so if there
4748 ** have been no vlan's registered do nothing.
4750 if (adapter->num_vlans == 0)
4753 /* Setup the queues for vlans */
4754 for (int i = 0; i < adapter->num_queues; i++) {
4755 rxr = &adapter->rx_rings[i];
4756 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4757 if (hw->mac.type != ixgbe_mac_82598EB) {
4758 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4759 ctrl |= IXGBE_RXDCTL_VME;
4760 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4762 rxr->vtag_strip = TRUE;
4765 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
4768 ** A soft reset zero's out the VFTA, so
4769 ** we need to repopulate it now.
4771 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4772 if (adapter->shadow_vfta[i] != 0)
4773 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4774 adapter->shadow_vfta[i]);
4776 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4777 /* Enable the Filter Table if enabled */
4778 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4779 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4780 ctrl |= IXGBE_VLNCTRL_VFE;
4782 if (hw->mac.type == ixgbe_mac_82598EB)
4783 ctrl |= IXGBE_VLNCTRL_VME;
4784 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4788 ixgbe_enable_intr(struct adapter *adapter)
4790 struct ixgbe_hw *hw = &adapter->hw;
4791 struct ix_queue *que = adapter->queues;
4794 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4795 /* Enable Fan Failure detection */
4796 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4797 mask |= IXGBE_EIMS_GPI_SDP1;
4799 switch (adapter->hw.mac.type) {
4800 case ixgbe_mac_82599EB:
4801 mask |= IXGBE_EIMS_ECC;
4802 mask |= IXGBE_EIMS_GPI_SDP0;
4803 mask |= IXGBE_EIMS_GPI_SDP1;
4804 mask |= IXGBE_EIMS_GPI_SDP2;
4806 mask |= IXGBE_EIMS_FLOW_DIR;
4809 case ixgbe_mac_X540:
4810 mask |= IXGBE_EIMS_ECC;
4811 /* Detect if Thermal Sensor is enabled */
4812 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
4813 if (fwsm & IXGBE_FWSM_TS_ENABLED)
4814 mask |= IXGBE_EIMS_TS;
4816 mask |= IXGBE_EIMS_FLOW_DIR;
4823 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4825 /* With RSS we use auto clear */
4826 if (adapter->msix_mem) {
4827 mask = IXGBE_EIMS_ENABLE_MASK;
4828 /* Don't autoclear Link */
4829 mask &= ~IXGBE_EIMS_OTHER;
4830 mask &= ~IXGBE_EIMS_LSC;
4831 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4835 ** Now enable all queues, this is done separately to
4836 ** allow for handling the extended (beyond 32) MSIX
4837 ** vectors that can be used by 82599
4839 for (int i = 0; i < adapter->num_queues; i++, que++)
4840 ixgbe_enable_queue(adapter, que->msix);
4842 IXGBE_WRITE_FLUSH(hw);
4848 ixgbe_disable_intr(struct adapter *adapter)
4850 if (adapter->msix_mem)
4851 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4852 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4853 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4855 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4856 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4857 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4859 IXGBE_WRITE_FLUSH(&adapter->hw);
4864 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4868 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4875 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4877 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4884 ** Get the width and transaction speed of
4885 ** the slot this adapter is plugged into.
4888 ixgbe_get_slot_info(struct ixgbe_hw *hw)
4890 device_t dev = ((struct ixgbe_osdep *)hw->back)->dev;
4891 struct ixgbe_mac_info *mac = &hw->mac;
4895 /* For most devices simply call the shared code routine */
4896 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
4897 ixgbe_get_bus_info(hw);
4902 ** For the Quad port adapter we need to parse back
4903 ** up the PCI tree to find the speed of the expansion
4904 ** slot into which this adapter is plugged. A bit more work.
4906 dev = device_get_parent(device_get_parent(dev));
4908 device_printf(dev, "parent pcib = %x,%x,%x\n",
4909 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4911 dev = device_get_parent(device_get_parent(dev));
4913 device_printf(dev, "slot pcib = %x,%x,%x\n",
4914 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4916 /* Now get the PCI Express Capabilities offset */
4917 pci_find_cap(dev, PCIY_EXPRESS, &offset);
4918 /* ...and read the Link Status Register */
4919 link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4920 switch (link & IXGBE_PCI_LINK_WIDTH) {
4921 case IXGBE_PCI_LINK_WIDTH_1:
4922 hw->bus.width = ixgbe_bus_width_pcie_x1;
4924 case IXGBE_PCI_LINK_WIDTH_2:
4925 hw->bus.width = ixgbe_bus_width_pcie_x2;
4927 case IXGBE_PCI_LINK_WIDTH_4:
4928 hw->bus.width = ixgbe_bus_width_pcie_x4;
4930 case IXGBE_PCI_LINK_WIDTH_8:
4931 hw->bus.width = ixgbe_bus_width_pcie_x8;
4934 hw->bus.width = ixgbe_bus_width_unknown;
4938 switch (link & IXGBE_PCI_LINK_SPEED) {
4939 case IXGBE_PCI_LINK_SPEED_2500:
4940 hw->bus.speed = ixgbe_bus_speed_2500;
4942 case IXGBE_PCI_LINK_SPEED_5000:
4943 hw->bus.speed = ixgbe_bus_speed_5000;
4945 case IXGBE_PCI_LINK_SPEED_8000:
4946 hw->bus.speed = ixgbe_bus_speed_8000;
4949 hw->bus.speed = ixgbe_bus_speed_unknown;
4953 mac->ops.set_lan_id(hw);
4956 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4957 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
4958 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
4959 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4960 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
4961 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
4962 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
4965 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4966 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
4967 (hw->bus.speed == ixgbe_bus_speed_2500))) {
4968 device_printf(dev, "PCI-Express bandwidth available"
4969 " for this card\n is not sufficient for"
4970 " optimal performance.\n");
4971 device_printf(dev, "For optimal performance a x8 "
4972 "PCIE, or x4 PCIE Gen2 slot is required.\n");
4974 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4975 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
4976 (hw->bus.speed < ixgbe_bus_speed_8000))) {
4977 device_printf(dev, "PCI-Express bandwidth available"
4978 " for this card\n is not sufficient for"
4979 " optimal performance.\n");
4980 device_printf(dev, "For optimal performance a x8 "
4981 "PCIE Gen3 slot is required.\n");
4989 ** Setup the correct IVAR register for a particular MSIX interrupt
4990 ** (yes this is all very magic and confusing :)
4991 ** - entry is the register array entry
4992 ** - vector is the MSIX vector for this queue
4993 ** - type is RX/TX/MISC
4996 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4998 struct ixgbe_hw *hw = &adapter->hw;
5001 vector |= IXGBE_IVAR_ALLOC_VAL;
5003 switch (hw->mac.type) {
5005 case ixgbe_mac_82598EB:
5007 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5009 entry += (type * 64);
5010 index = (entry >> 2) & 0x1F;
5011 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5012 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5013 ivar |= (vector << (8 * (entry & 0x3)));
5014 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5017 case ixgbe_mac_82599EB:
5018 case ixgbe_mac_X540:
5019 if (type == -1) { /* MISC IVAR */
5020 index = (entry & 1) * 8;
5021 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5022 ivar &= ~(0xFF << index);
5023 ivar |= (vector << index);
5024 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5025 } else { /* RX/TX IVARS */
5026 index = (16 * (entry & 1)) + (8 * type);
5027 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5028 ivar &= ~(0xFF << index);
5029 ivar |= (vector << index);
5030 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5039 ixgbe_configure_ivars(struct adapter *adapter)
5041 struct ix_queue *que = adapter->queues;
5044 if (ixgbe_max_interrupt_rate > 0)
5045 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5049 for (int i = 0; i < adapter->num_queues; i++, que++) {
5050 /* First the RX queue entry */
5051 ixgbe_set_ivar(adapter, i, que->msix, 0);
5052 /* ... and the TX */
5053 ixgbe_set_ivar(adapter, i, que->msix, 1);
5054 /* Set an Initial EITR value */
5055 IXGBE_WRITE_REG(&adapter->hw,
5056 IXGBE_EITR(que->msix), newitr);
5059 /* For the Link interrupt */
5060 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5064 ** ixgbe_sfp_probe - called in the local timer to
5065 ** determine if a port had optics inserted.
5067 static bool ixgbe_sfp_probe(struct adapter *adapter)
5069 struct ixgbe_hw *hw = &adapter->hw;
5070 device_t dev = adapter->dev;
5071 bool result = FALSE;
5073 if ((hw->phy.type == ixgbe_phy_nl) &&
5074 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5075 s32 ret = hw->phy.ops.identify_sfp(hw);
5078 ret = hw->phy.ops.reset(hw);
5079 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5080 device_printf(dev,"Unsupported SFP+ module detected!");
5081 printf(" Reload driver with supported module.\n");
5082 adapter->sfp_probe = FALSE;
5085 device_printf(dev,"SFP+ module detected!\n");
5086 /* We now have supported optics */
5087 adapter->sfp_probe = FALSE;
5088 /* Set the optics type so system reports correctly */
5089 ixgbe_setup_optics(adapter);
5097 ** Tasklet handler for MSIX Link interrupts
5098 ** - do outside interrupt since it might sleep
5101 ixgbe_handle_link(void *context, int pending)
5103 struct adapter *adapter = context;
5105 ixgbe_check_link(&adapter->hw,
5106 &adapter->link_speed, &adapter->link_up, 0);
5107 ixgbe_update_link_status(adapter);
5111 ** Tasklet for handling SFP module interrupts
5114 ixgbe_handle_mod(void *context, int pending)
5116 struct adapter *adapter = context;
5117 struct ixgbe_hw *hw = &adapter->hw;
5118 device_t dev = adapter->dev;
5121 err = hw->phy.ops.identify_sfp(hw);
5122 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5124 "Unsupported SFP+ module type was detected.\n");
5127 err = hw->mac.ops.setup_sfp(hw);
5128 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5130 "Setup failure - unsupported SFP+ module type.\n");
5133 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5139 ** Tasklet for handling MSF (multispeed fiber) interrupts
5142 ixgbe_handle_msf(void *context, int pending)
5144 struct adapter *adapter = context;
5145 struct ixgbe_hw *hw = &adapter->hw;
5149 autoneg = hw->phy.autoneg_advertised;
5150 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5151 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5152 if (hw->mac.ops.setup_link)
5153 hw->mac.ops.setup_link(hw, autoneg, TRUE);
5159 ** Tasklet for reinitializing the Flow Director filter table
5162 ixgbe_reinit_fdir(void *context, int pending)
5164 struct adapter *adapter = context;
5165 struct ifnet *ifp = adapter->ifp;
5167 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5169 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5170 adapter->fdir_reinit = 0;
5171 /* re-enable flow director interrupts */
5172 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5173 /* Restart the interface */
5174 ifp->if_drv_flags |= IFF_DRV_RUNNING;
5179 /**********************************************************************
5181 * Update the board statistics counters.
5183 **********************************************************************/
5185 ixgbe_update_stats_counters(struct adapter *adapter)
5187 struct ifnet *ifp = adapter->ifp;
5188 struct ixgbe_hw *hw = &adapter->hw;
5189 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5190 u64 total_missed_rx = 0;
5192 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5193 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5194 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5195 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5198 ** Note: these are for the 8 possible traffic classes,
5199 ** which in current implementation is unused,
5200 ** therefore only 0 should read real data.
5202 for (int i = 0; i < 8; i++) {
5204 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5205 /* missed_rx tallies misses for the gprc workaround */
5207 /* global total per queue */
5208 adapter->stats.mpc[i] += mp;
5209 /* Running comprehensive total for stats display */
5210 total_missed_rx += adapter->stats.mpc[i];
5211 if (hw->mac.type == ixgbe_mac_82598EB) {
5212 adapter->stats.rnbc[i] +=
5213 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5214 adapter->stats.qbtc[i] +=
5215 IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5216 adapter->stats.qbrc[i] +=
5217 IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5218 adapter->stats.pxonrxc[i] +=
5219 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5221 adapter->stats.pxonrxc[i] +=
5222 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5223 adapter->stats.pxontxc[i] +=
5224 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5225 adapter->stats.pxofftxc[i] +=
5226 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5227 adapter->stats.pxoffrxc[i] +=
5228 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5229 adapter->stats.pxon2offc[i] +=
5230 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5232 for (int i = 0; i < 16; i++) {
5233 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5234 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5235 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5237 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5238 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5239 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5241 /* Hardware workaround, gprc counts missed packets */
5242 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5243 adapter->stats.gprc -= missed_rx;
5245 if (hw->mac.type != ixgbe_mac_82598EB) {
5246 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5247 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5248 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5249 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5250 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5251 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5252 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5253 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5255 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5256 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5257 /* 82598 only has a counter in the high register */
5258 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5259 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5260 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5264 * Workaround: mprc hardware is incorrectly counting
5265 * broadcasts, so for now we subtract those.
5267 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5268 adapter->stats.bprc += bprc;
5269 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5270 if (hw->mac.type == ixgbe_mac_82598EB)
5271 adapter->stats.mprc -= bprc;
5273 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5274 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5275 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5276 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5277 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5278 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5280 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5281 adapter->stats.lxontxc += lxon;
5282 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5283 adapter->stats.lxofftxc += lxoff;
5284 total = lxon + lxoff;
5286 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5287 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5288 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5289 adapter->stats.gptc -= total;
5290 adapter->stats.mptc -= total;
5291 adapter->stats.ptc64 -= total;
5292 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5294 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5295 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5296 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5297 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5298 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5299 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5300 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5301 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5302 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5303 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5304 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5305 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5306 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5307 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5308 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5309 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5310 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5311 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5312 /* Only read FCOE on 82599 */
5313 if (hw->mac.type != ixgbe_mac_82598EB) {
5314 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5315 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5316 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5317 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5318 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5321 /* Fill out the OS statistics structure */
5322 ifp->if_ipackets = adapter->stats.gprc;
5323 ifp->if_opackets = adapter->stats.gptc;
5324 ifp->if_ibytes = adapter->stats.gorc;
5325 ifp->if_obytes = adapter->stats.gotc;
5326 ifp->if_imcasts = adapter->stats.mprc;
5327 ifp->if_omcasts = adapter->stats.mptc;
5328 ifp->if_collisions = 0;
5331 ifp->if_iqdrops = total_missed_rx;
5332 ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec;
5335 /** ixgbe_sysctl_tdh_handler - Handler function
5336 * Retrieves the TDH value from the hardware
5339 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5343 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5346 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5347 error = sysctl_handle_int(oidp, &val, 0, req);
5348 if (error || !req->newptr)
5353 /** ixgbe_sysctl_tdt_handler - Handler function
5354 * Retrieves the TDT value from the hardware
5357 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5361 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5364 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5365 error = sysctl_handle_int(oidp, &val, 0, req);
5366 if (error || !req->newptr)
5371 /** ixgbe_sysctl_rdh_handler - Handler function
5372 * Retrieves the RDH value from the hardware
5375 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5379 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5382 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5383 error = sysctl_handle_int(oidp, &val, 0, req);
5384 if (error || !req->newptr)
5389 /** ixgbe_sysctl_rdt_handler - Handler function
5390 * Retrieves the RDT value from the hardware
5393 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5397 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5400 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5401 error = sysctl_handle_int(oidp, &val, 0, req);
5402 if (error || !req->newptr)
5408 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5411 struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5412 unsigned int reg, usec, rate;
5414 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5415 usec = ((reg & 0x0FF8) >> 3);
5417 rate = 500000 / usec;
5420 error = sysctl_handle_int(oidp, &rate, 0, req);
5421 if (error || !req->newptr)
5423 reg &= ~0xfff; /* default, no limitation */
5424 ixgbe_max_interrupt_rate = 0;
5425 if (rate > 0 && rate < 500000) {
5428 ixgbe_max_interrupt_rate = rate;
5429 reg |= ((4000000/rate) & 0xff8 );
5431 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5436 * Add sysctl variables, one per statistic, to the system.
5439 ixgbe_add_hw_stats(struct adapter *adapter)
5442 device_t dev = adapter->dev;
5444 struct tx_ring *txr = adapter->tx_rings;
5445 struct rx_ring *rxr = adapter->rx_rings;
5447 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5448 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5449 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5450 struct ixgbe_hw_stats *stats = &adapter->stats;
5452 struct sysctl_oid *stat_node, *queue_node;
5453 struct sysctl_oid_list *stat_list, *queue_list;
5455 #define QUEUE_NAME_LEN 32
5456 char namebuf[QUEUE_NAME_LEN];
5458 /* Driver Statistics */
5459 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5460 CTLFLAG_RD, &adapter->dropped_pkts,
5461 "Driver dropped packets");
5462 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5463 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5464 "m_defrag() failed");
5465 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5466 CTLFLAG_RD, &adapter->watchdog_events,
5467 "Watchdog timeouts");
5468 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5469 CTLFLAG_RD, &adapter->link_irq,
5470 "Link MSIX IRQ Handled");
5472 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5473 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5474 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5475 CTLFLAG_RD, NULL, "Queue Name");
5476 queue_list = SYSCTL_CHILDREN(queue_node);
5478 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5479 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5480 sizeof(&adapter->queues[i]),
5481 ixgbe_sysctl_interrupt_rate_handler, "IU",
5483 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5484 CTLFLAG_RD, &(adapter->queues[i].irqs),
5485 "irqs on this queue");
5486 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5487 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5488 ixgbe_sysctl_tdh_handler, "IU",
5489 "Transmit Descriptor Head");
5490 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5491 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5492 ixgbe_sysctl_tdt_handler, "IU",
5493 "Transmit Descriptor Tail");
5494 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5495 CTLFLAG_RD, &txr->tso_tx,
5497 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5498 CTLFLAG_RD, &txr->no_tx_dma_setup,
5499 "Driver tx dma failure in xmit");
5500 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5501 CTLFLAG_RD, &txr->no_desc_avail,
5502 "Queue No Descriptor Available");
5503 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5504 CTLFLAG_RD, &txr->total_packets,
5505 "Queue Packets Transmitted");
5508 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5509 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5510 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5511 CTLFLAG_RD, NULL, "Queue Name");
5512 queue_list = SYSCTL_CHILDREN(queue_node);
5514 struct lro_ctrl *lro = &rxr->lro;
5516 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5517 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5518 CTLFLAG_RD, NULL, "Queue Name");
5519 queue_list = SYSCTL_CHILDREN(queue_node);
5521 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5522 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5523 ixgbe_sysctl_rdh_handler, "IU",
5524 "Receive Descriptor Head");
5525 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5526 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5527 ixgbe_sysctl_rdt_handler, "IU",
5528 "Receive Descriptor Tail");
5529 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5530 CTLFLAG_RD, &rxr->rx_packets,
5531 "Queue Packets Received");
5532 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5533 CTLFLAG_RD, &rxr->rx_bytes,
5534 "Queue Bytes Received");
5535 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5536 CTLFLAG_RD, &rxr->rx_copies,
5537 "Copied RX Frames");
5538 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5539 CTLFLAG_RD, &lro->lro_queued, 0,
5541 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5542 CTLFLAG_RD, &lro->lro_flushed, 0,
5546 /* MAC stats get the own sub node */
5548 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5549 CTLFLAG_RD, NULL, "MAC Statistics");
5550 stat_list = SYSCTL_CHILDREN(stat_node);
5552 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5553 CTLFLAG_RD, &stats->crcerrs,
5555 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5556 CTLFLAG_RD, &stats->illerrc,
5557 "Illegal Byte Errors");
5558 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5559 CTLFLAG_RD, &stats->errbc,
5561 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5562 CTLFLAG_RD, &stats->mspdc,
5563 "MAC Short Packets Discarded");
5564 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5565 CTLFLAG_RD, &stats->mlfc,
5566 "MAC Local Faults");
5567 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5568 CTLFLAG_RD, &stats->mrfc,
5569 "MAC Remote Faults");
5570 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5571 CTLFLAG_RD, &stats->rlec,
5572 "Receive Length Errors");
5574 /* Flow Control stats */
5575 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5576 CTLFLAG_RD, &stats->lxontxc,
5577 "Link XON Transmitted");
5578 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5579 CTLFLAG_RD, &stats->lxonrxc,
5580 "Link XON Received");
5581 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5582 CTLFLAG_RD, &stats->lxofftxc,
5583 "Link XOFF Transmitted");
5584 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5585 CTLFLAG_RD, &stats->lxoffrxc,
5586 "Link XOFF Received");
5588 /* Packet Reception Stats */
5589 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5590 CTLFLAG_RD, &stats->tor,
5591 "Total Octets Received");
5592 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5593 CTLFLAG_RD, &stats->gorc,
5594 "Good Octets Received");
5595 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5596 CTLFLAG_RD, &stats->tpr,
5597 "Total Packets Received");
5598 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5599 CTLFLAG_RD, &stats->gprc,
5600 "Good Packets Received");
5601 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5602 CTLFLAG_RD, &stats->mprc,
5603 "Multicast Packets Received");
5604 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5605 CTLFLAG_RD, &stats->bprc,
5606 "Broadcast Packets Received");
5607 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5608 CTLFLAG_RD, &stats->prc64,
5609 "64 byte frames received ");
5610 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5611 CTLFLAG_RD, &stats->prc127,
5612 "65-127 byte frames received");
5613 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5614 CTLFLAG_RD, &stats->prc255,
5615 "128-255 byte frames received");
5616 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5617 CTLFLAG_RD, &stats->prc511,
5618 "256-511 byte frames received");
5619 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5620 CTLFLAG_RD, &stats->prc1023,
5621 "512-1023 byte frames received");
5622 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5623 CTLFLAG_RD, &stats->prc1522,
5624 "1023-1522 byte frames received");
5625 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5626 CTLFLAG_RD, &stats->ruc,
5627 "Receive Undersized");
5628 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5629 CTLFLAG_RD, &stats->rfc,
5630 "Fragmented Packets Received ");
5631 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5632 CTLFLAG_RD, &stats->roc,
5633 "Oversized Packets Received");
5634 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5635 CTLFLAG_RD, &stats->rjc,
5637 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5638 CTLFLAG_RD, &stats->mngprc,
5639 "Management Packets Received");
5640 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5641 CTLFLAG_RD, &stats->mngptc,
5642 "Management Packets Dropped");
5643 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5644 CTLFLAG_RD, &stats->xec,
5647 /* Packet Transmission Stats */
5648 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5649 CTLFLAG_RD, &stats->gotc,
5650 "Good Octets Transmitted");
5651 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5652 CTLFLAG_RD, &stats->tpt,
5653 "Total Packets Transmitted");
5654 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5655 CTLFLAG_RD, &stats->gptc,
5656 "Good Packets Transmitted");
5657 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5658 CTLFLAG_RD, &stats->bptc,
5659 "Broadcast Packets Transmitted");
5660 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5661 CTLFLAG_RD, &stats->mptc,
5662 "Multicast Packets Transmitted");
5663 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5664 CTLFLAG_RD, &stats->mngptc,
5665 "Management Packets Transmitted");
5666 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5667 CTLFLAG_RD, &stats->ptc64,
5668 "64 byte frames transmitted ");
5669 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5670 CTLFLAG_RD, &stats->ptc127,
5671 "65-127 byte frames transmitted");
5672 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5673 CTLFLAG_RD, &stats->ptc255,
5674 "128-255 byte frames transmitted");
5675 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5676 CTLFLAG_RD, &stats->ptc511,
5677 "256-511 byte frames transmitted");
5678 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5679 CTLFLAG_RD, &stats->ptc1023,
5680 "512-1023 byte frames transmitted");
5681 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5682 CTLFLAG_RD, &stats->ptc1522,
5683 "1024-1522 byte frames transmitted");
5687 ** Set flow control using sysctl:
5688 ** Flow control values:
5695 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5698 struct adapter *adapter = (struct adapter *) arg1;
5701 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5702 if ((error) || (req->newptr == NULL))
5705 /* Don't bother if it's not changed */
5706 if (adapter->fc == last)
5709 switch (adapter->fc) {
5710 case ixgbe_fc_rx_pause:
5711 case ixgbe_fc_tx_pause:
5713 adapter->hw.fc.requested_mode = adapter->fc;
5714 if (adapter->num_queues > 1)
5715 ixgbe_disable_rx_drop(adapter);
5718 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5719 if (adapter->num_queues > 1)
5720 ixgbe_enable_rx_drop(adapter);
5726 /* Don't autoneg if forcing a value */
5727 adapter->hw.fc.disable_fc_autoneg = TRUE;
5728 ixgbe_fc_enable(&adapter->hw);
5733 ** Control link advertise speed:
5734 ** 1 - advertise only 1G
5735 ** 2 - advertise 100Mb
5736 ** 3 - advertise normal
5739 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5742 struct adapter *adapter;
5744 struct ixgbe_hw *hw;
5745 ixgbe_link_speed speed, last;
5747 adapter = (struct adapter *) arg1;
5750 last = adapter->advertise;
5752 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5753 if ((error) || (req->newptr == NULL))
5756 if (adapter->advertise == last) /* no change */
5759 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5760 (hw->phy.multispeed_fiber)))
5763 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5764 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5768 if (adapter->advertise == 1)
5769 speed = IXGBE_LINK_SPEED_1GB_FULL;
5770 else if (adapter->advertise == 2)
5771 speed = IXGBE_LINK_SPEED_100_FULL;
5772 else if (adapter->advertise == 3)
5773 speed = IXGBE_LINK_SPEED_1GB_FULL |
5774 IXGBE_LINK_SPEED_10GB_FULL;
5775 else { /* bogus value */
5776 adapter->advertise = last;
5780 hw->mac.autotry_restart = TRUE;
5781 hw->mac.ops.setup_link(hw, speed, TRUE);
5787 ** Thermal Shutdown Trigger
5788 ** - cause a Thermal Overtemp IRQ
5789 ** - this now requires firmware enabling
5792 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5794 int error, fire = 0;
5795 struct adapter *adapter = (struct adapter *) arg1;
5796 struct ixgbe_hw *hw = &adapter->hw;
5799 if (hw->mac.type != ixgbe_mac_X540)
5802 error = sysctl_handle_int(oidp, &fire, 0, req);
5803 if ((error) || (req->newptr == NULL))
5807 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5808 reg |= IXGBE_EICR_TS;
5809 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5816 ** Enable the hardware to drop packets when the buffer is
5817 ** full. This is useful when multiqueue,so that no single
5818 ** queue being full stalls the entire RX engine. We only
5819 ** enable this when Multiqueue AND when Flow Control is
5823 ixgbe_enable_rx_drop(struct adapter *adapter)
5825 struct ixgbe_hw *hw = &adapter->hw;
5827 for (int i = 0; i < adapter->num_queues; i++) {
5828 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5829 srrctl |= IXGBE_SRRCTL_DROP_EN;
5830 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5835 ixgbe_disable_rx_drop(struct adapter *adapter)
5837 struct ixgbe_hw *hw = &adapter->hw;
5839 for (int i = 0; i < adapter->num_queues; i++) {
5840 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5841 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5842 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);