1 /******************************************************************************
3 Copyright (c) 2001-2013, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.5.13";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 #ifdef IXGBE_LEGACY_TX
106 static void ixgbe_start(struct ifnet *);
107 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
108 #else /* ! IXGBE_LEGACY_TX */
109 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
110 static int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
111 static void ixgbe_qflush(struct ifnet *);
112 static void ixgbe_deferred_mq_start(void *, int);
113 #endif /* IXGBE_LEGACY_TX */
114 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
115 static void ixgbe_init(void *);
116 static void ixgbe_init_locked(struct adapter *);
117 static void ixgbe_stop(void *);
118 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
119 static int ixgbe_media_change(struct ifnet *);
120 static void ixgbe_identify_hardware(struct adapter *);
121 static int ixgbe_allocate_pci_resources(struct adapter *);
122 static void ixgbe_get_slot_info(struct ixgbe_hw *);
123 static int ixgbe_allocate_msix(struct adapter *);
124 static int ixgbe_allocate_legacy(struct adapter *);
125 static int ixgbe_allocate_queues(struct adapter *);
126 static int ixgbe_setup_msix(struct adapter *);
127 static void ixgbe_free_pci_resources(struct adapter *);
128 static void ixgbe_local_timer(void *);
129 static int ixgbe_setup_interface(device_t, struct adapter *);
130 static void ixgbe_config_link(struct adapter *);
132 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133 static int ixgbe_setup_transmit_structures(struct adapter *);
134 static void ixgbe_setup_transmit_ring(struct tx_ring *);
135 static void ixgbe_initialize_transmit_units(struct adapter *);
136 static void ixgbe_free_transmit_structures(struct adapter *);
137 static void ixgbe_free_transmit_buffers(struct tx_ring *);
139 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140 static int ixgbe_setup_receive_structures(struct adapter *);
141 static int ixgbe_setup_receive_ring(struct rx_ring *);
142 static void ixgbe_initialize_receive_units(struct adapter *);
143 static void ixgbe_free_receive_structures(struct adapter *);
144 static void ixgbe_free_receive_buffers(struct rx_ring *);
145 static void ixgbe_setup_hw_rsc(struct rx_ring *);
147 static void ixgbe_enable_intr(struct adapter *);
148 static void ixgbe_disable_intr(struct adapter *);
149 static void ixgbe_update_stats_counters(struct adapter *);
150 static void ixgbe_txeof(struct tx_ring *);
151 static bool ixgbe_rxeof(struct ix_queue *);
152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void ixgbe_set_promisc(struct adapter *);
154 static void ixgbe_set_multi(struct adapter *);
155 static void ixgbe_update_link_status(struct adapter *);
156 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
162 struct ixgbe_dma_alloc *, int);
163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static int ixgbe_tx_ctx_setup(struct tx_ring *,
165 struct mbuf *, u32 *, u32 *);
166 static int ixgbe_tso_setup(struct tx_ring *,
167 struct mbuf *, u32 *, u32 *);
168 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
169 static void ixgbe_configure_ivars(struct adapter *);
170 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
172 static void ixgbe_setup_vlan_hw_support(struct adapter *);
173 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
174 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
176 static void ixgbe_add_hw_stats(struct adapter *adapter);
178 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
179 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
182 static void ixgbe_enable_rx_drop(struct adapter *);
183 static void ixgbe_disable_rx_drop(struct adapter *);
185 /* Support for pluggable optic modules */
186 static bool ixgbe_sfp_probe(struct adapter *);
187 static void ixgbe_setup_optics(struct adapter *);
189 /* Legacy (single vector interrupt handler */
190 static void ixgbe_legacy_irq(void *);
192 /* The MSI/X Interrupt handlers */
193 static void ixgbe_msix_que(void *);
194 static void ixgbe_msix_link(void *);
196 /* Deferred interrupt tasklets */
197 static void ixgbe_handle_que(void *, int);
198 static void ixgbe_handle_link(void *, int);
199 static void ixgbe_handle_msf(void *, int);
200 static void ixgbe_handle_mod(void *, int);
203 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
204 static void ixgbe_reinit_fdir(void *, int);
207 /* Missing shared code prototype */
208 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
210 /*********************************************************************
211 * FreeBSD Device Interface Entry Points
212 *********************************************************************/
214 static device_method_t ixgbe_methods[] = {
215 /* Device interface */
216 DEVMETHOD(device_probe, ixgbe_probe),
217 DEVMETHOD(device_attach, ixgbe_attach),
218 DEVMETHOD(device_detach, ixgbe_detach),
219 DEVMETHOD(device_shutdown, ixgbe_shutdown),
223 static driver_t ixgbe_driver = {
224 "ix", ixgbe_methods, sizeof(struct adapter),
227 devclass_t ixgbe_devclass;
228 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
230 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
231 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
234 ** TUNEABLE PARAMETERS:
238 ** AIM: Adaptive Interrupt Moderation
239 ** which means that the interrupt rate
240 ** is varied over time based on the
241 ** traffic for that interrupt vector
243 static int ixgbe_enable_aim = TRUE;
244 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
246 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
247 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
249 /* How many packets rxeof tries to clean at a time */
250 static int ixgbe_rx_process_limit = 256;
251 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
253 /* How many packets txeof tries to clean at a time */
254 static int ixgbe_tx_process_limit = 256;
255 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
258 ** Smart speed setting, default to on
259 ** this only works as a compile option
260 ** right now as its during attach, set
261 ** this to 'ixgbe_smart_speed_off' to
264 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
267 * MSIX should be the default for best performance,
268 * but this allows it to be forced off for testing.
270 static int ixgbe_enable_msix = 1;
271 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
274 * Number of Queues, can be set to 0,
275 * it then autoconfigures based on the
276 * number of cpus with a max of 8. This
277 * can be overriden manually here.
279 static int ixgbe_num_queues = 0;
280 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
283 ** Number of TX descriptors per ring,
284 ** setting higher than RX as this seems
285 ** the better performing choice.
287 static int ixgbe_txd = PERFORM_TXD;
288 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
290 /* Number of RX descriptors per ring */
291 static int ixgbe_rxd = PERFORM_RXD;
292 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
295 ** Defining this on will allow the use
296 ** of unsupported SFP+ modules, note that
297 ** doing so you are on your own :)
299 static int allow_unsupported_sfp = FALSE;
303 ** this feature only works with
304 ** IPv4, and only on 82599 and later.
305 ** Also this will cause IP forwarding to
306 ** fail and that can't be controlled by
307 ** the stack as LRO can. For all these
308 ** reasons I've deemed it best to leave
309 ** this off and not bother with a tuneable
310 ** interface, this would need to be compiled
313 static bool ixgbe_rsc_enable = FALSE;
315 /* Keep running tab on them for sanity check */
316 static int ixgbe_total_ports;
320 ** For Flow Director: this is the
321 ** number of TX packets we sample
322 ** for the filter pool, this means
323 ** every 20th packet will be probed.
325 ** This feature can be disabled by
326 ** setting this to 0.
328 static int atr_sample_rate = 20;
330 ** Flow Director actually 'steals'
331 ** part of the packet buffer as its
332 ** filter pool, this variable controls
334 ** 0 = 64K, 1 = 128K, 2 = 256K
336 static int fdir_pballoc = 1;
341 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
342 * be a reference on how to implement netmap support in a driver.
343 * Additional comments are in ixgbe_netmap.h .
345 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
346 * that extend the standard driver.
348 #include <dev/netmap/ixgbe_netmap.h>
349 #endif /* DEV_NETMAP */
351 /*********************************************************************
352 * Device identification routine
354 * ixgbe_probe determines if the driver should be loaded on
355 * adapter based on PCI vendor/device id of the adapter.
357 * return BUS_PROBE_DEFAULT on success, positive on failure
358 *********************************************************************/
361 ixgbe_probe(device_t dev)
363 ixgbe_vendor_info_t *ent;
365 u16 pci_vendor_id = 0;
366 u16 pci_device_id = 0;
367 u16 pci_subvendor_id = 0;
368 u16 pci_subdevice_id = 0;
369 char adapter_name[256];
371 INIT_DEBUGOUT("ixgbe_probe: begin");
373 pci_vendor_id = pci_get_vendor(dev);
374 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
377 pci_device_id = pci_get_device(dev);
378 pci_subvendor_id = pci_get_subvendor(dev);
379 pci_subdevice_id = pci_get_subdevice(dev);
381 ent = ixgbe_vendor_info_array;
382 while (ent->vendor_id != 0) {
383 if ((pci_vendor_id == ent->vendor_id) &&
384 (pci_device_id == ent->device_id) &&
386 ((pci_subvendor_id == ent->subvendor_id) ||
387 (ent->subvendor_id == 0)) &&
389 ((pci_subdevice_id == ent->subdevice_id) ||
390 (ent->subdevice_id == 0))) {
391 sprintf(adapter_name, "%s, Version - %s",
392 ixgbe_strings[ent->index],
393 ixgbe_driver_version);
394 device_set_desc_copy(dev, adapter_name);
396 return (BUS_PROBE_DEFAULT);
403 /*********************************************************************
404 * Device initialization routine
406 * The attach entry point is called when the driver is being loaded.
407 * This routine identifies the type of hardware, allocates all resources
408 * and initializes the hardware.
410 * return 0 on success, positive on failure
411 *********************************************************************/
414 ixgbe_attach(device_t dev)
416 struct adapter *adapter;
422 INIT_DEBUGOUT("ixgbe_attach: begin");
424 /* Allocate, clear, and link in our adapter structure */
425 adapter = device_get_softc(dev);
426 adapter->dev = adapter->osdep.dev = dev;
430 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
434 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
435 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
436 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
437 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
439 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
440 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
441 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
442 &ixgbe_enable_aim, 1, "Interrupt Moderation");
445 ** Allow a kind of speed control by forcing the autoneg
446 ** advertised speed list to only a certain value, this
447 ** supports 1G on 82599 devices, and 100Mb on x540.
449 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
450 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
451 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
452 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
454 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
455 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
456 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
457 0, ixgbe_set_thermal_test, "I", "Thermal Test");
459 /* Set up the timer callout */
460 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
462 /* Determine hardware revision */
463 ixgbe_identify_hardware(adapter);
465 /* Do base PCI setup - map BAR0 */
466 if (ixgbe_allocate_pci_resources(adapter)) {
467 device_printf(dev, "Allocation of PCI resources failed\n");
472 /* Do descriptor calc and sanity checks */
473 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
474 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
475 device_printf(dev, "TXD config issue, using default!\n");
476 adapter->num_tx_desc = DEFAULT_TXD;
478 adapter->num_tx_desc = ixgbe_txd;
481 ** With many RX rings it is easy to exceed the
482 ** system mbuf allocation. Tuning nmbclusters
483 ** can alleviate this.
485 if (nmbclusters > 0 ) {
487 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
488 if (s > nmbclusters) {
489 device_printf(dev, "RX Descriptors exceed "
490 "system mbuf max, using default instead!\n");
491 ixgbe_rxd = DEFAULT_RXD;
495 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
496 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
497 device_printf(dev, "RXD config issue, using default!\n");
498 adapter->num_rx_desc = DEFAULT_RXD;
500 adapter->num_rx_desc = ixgbe_rxd;
502 /* Allocate our TX/RX Queues */
503 if (ixgbe_allocate_queues(adapter)) {
508 /* Allocate multicast array memory. */
509 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
510 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
511 if (adapter->mta == NULL) {
512 device_printf(dev, "Can not allocate multicast setup array\n");
517 /* Initialize the shared code */
518 hw->allow_unsupported_sfp = allow_unsupported_sfp;
519 error = ixgbe_init_shared_code(hw);
520 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
522 ** No optics in this port, set up
523 ** so the timer routine will probe
524 ** for later insertion.
526 adapter->sfp_probe = TRUE;
528 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
529 device_printf(dev,"Unsupported SFP+ module detected!\n");
533 device_printf(dev,"Unable to initialize the shared code\n");
538 /* Make sure we have a good EEPROM before we read from it */
539 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
540 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
545 error = ixgbe_init_hw(hw);
547 case IXGBE_ERR_EEPROM_VERSION:
548 device_printf(dev, "This device is a pre-production adapter/"
549 "LOM. Please be aware there may be issues associated "
550 "with your hardware.\n If you are experiencing problems "
551 "please contact your Intel or hardware representative "
552 "who provided you with this hardware.\n");
554 case IXGBE_ERR_SFP_NOT_SUPPORTED:
555 device_printf(dev,"Unsupported SFP+ Module\n");
558 case IXGBE_ERR_SFP_NOT_PRESENT:
559 device_printf(dev,"No SFP+ Module found\n");
565 /* Detect and set physical type */
566 ixgbe_setup_optics(adapter);
568 if ((adapter->msix > 1) && (ixgbe_enable_msix))
569 error = ixgbe_allocate_msix(adapter);
571 error = ixgbe_allocate_legacy(adapter);
575 /* Setup OS specific network interface */
576 if (ixgbe_setup_interface(dev, adapter) != 0)
579 /* Initialize statistics */
580 ixgbe_update_stats_counters(adapter);
582 /* Register for VLAN events */
583 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
584 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
585 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
586 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
589 ** Check PCIE slot type/speed/width
591 ixgbe_get_slot_info(hw);
593 /* Set an initial default flow control value */
594 adapter->fc = ixgbe_fc_full;
596 /* let hardware know driver is loaded */
597 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
598 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
599 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
601 ixgbe_add_hw_stats(adapter);
604 ixgbe_netmap_attach(adapter);
605 #endif /* DEV_NETMAP */
606 INIT_DEBUGOUT("ixgbe_attach: end");
609 ixgbe_free_transmit_structures(adapter);
610 ixgbe_free_receive_structures(adapter);
612 if (adapter->ifp != NULL)
613 if_free(adapter->ifp);
614 ixgbe_free_pci_resources(adapter);
615 free(adapter->mta, M_DEVBUF);
620 /*********************************************************************
621 * Device removal routine
623 * The detach entry point is called when the driver is being removed.
624 * This routine stops the adapter and deallocates all the resources
625 * that were allocated for driver operation.
627 * return 0 on success, positive on failure
628 *********************************************************************/
631 ixgbe_detach(device_t dev)
633 struct adapter *adapter = device_get_softc(dev);
634 struct ix_queue *que = adapter->queues;
635 struct tx_ring *txr = adapter->tx_rings;
638 INIT_DEBUGOUT("ixgbe_detach: begin");
640 /* Make sure VLANS are not using driver */
641 if (adapter->ifp->if_vlantrunk != NULL) {
642 device_printf(dev,"Vlan in use, detach first\n");
646 IXGBE_CORE_LOCK(adapter);
648 IXGBE_CORE_UNLOCK(adapter);
650 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
652 #ifndef IXGBE_LEGACY_TX
653 taskqueue_drain(que->tq, &txr->txq_task);
655 taskqueue_drain(que->tq, &que->que_task);
656 taskqueue_free(que->tq);
660 /* Drain the Link queue */
662 taskqueue_drain(adapter->tq, &adapter->link_task);
663 taskqueue_drain(adapter->tq, &adapter->mod_task);
664 taskqueue_drain(adapter->tq, &adapter->msf_task);
666 taskqueue_drain(adapter->tq, &adapter->fdir_task);
668 taskqueue_free(adapter->tq);
671 /* let hardware know driver is unloading */
672 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
673 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
674 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
676 /* Unregister VLAN events */
677 if (adapter->vlan_attach != NULL)
678 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
679 if (adapter->vlan_detach != NULL)
680 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
682 ether_ifdetach(adapter->ifp);
683 callout_drain(&adapter->timer);
685 netmap_detach(adapter->ifp);
686 #endif /* DEV_NETMAP */
687 ixgbe_free_pci_resources(adapter);
688 bus_generic_detach(dev);
689 if_free(adapter->ifp);
691 ixgbe_free_transmit_structures(adapter);
692 ixgbe_free_receive_structures(adapter);
693 free(adapter->mta, M_DEVBUF);
695 IXGBE_CORE_LOCK_DESTROY(adapter);
699 /*********************************************************************
701 * Shutdown entry point
703 **********************************************************************/
706 ixgbe_shutdown(device_t dev)
708 struct adapter *adapter = device_get_softc(dev);
709 IXGBE_CORE_LOCK(adapter);
711 IXGBE_CORE_UNLOCK(adapter);
716 #ifdef IXGBE_LEGACY_TX
717 /*********************************************************************
718 * Transmit entry point
720 * ixgbe_start is called by the stack to initiate a transmit.
721 * The driver will remain in this routine as long as there are
722 * packets to transmit and transmit resources are available.
723 * In case resources are not available stack is notified and
724 * the packet is requeued.
725 **********************************************************************/
728 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
731 struct adapter *adapter = txr->adapter;
733 IXGBE_TX_LOCK_ASSERT(txr);
735 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
737 if (!adapter->link_active)
740 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
741 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
744 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
748 if (ixgbe_xmit(txr, &m_head)) {
750 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
753 /* Send a copy of the frame to the BPF listener */
754 ETHER_BPF_MTAP(ifp, m_head);
756 /* Set watchdog on */
757 txr->watchdog_time = ticks;
758 txr->queue_status = IXGBE_QUEUE_WORKING;
765 * Legacy TX start - called by the stack, this
766 * always uses the first tx ring, and should
767 * not be used with multiqueue tx enabled.
770 ixgbe_start(struct ifnet *ifp)
772 struct adapter *adapter = ifp->if_softc;
773 struct tx_ring *txr = adapter->tx_rings;
775 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
777 ixgbe_start_locked(txr, ifp);
778 IXGBE_TX_UNLOCK(txr);
783 #else /* ! IXGBE_LEGACY_TX */
786 ** Multiqueue Transmit driver
790 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
792 struct adapter *adapter = ifp->if_softc;
793 struct ix_queue *que;
797 /* Which queue to use */
798 if ((m->m_flags & M_FLOWID) != 0)
799 i = m->m_pkthdr.flowid % adapter->num_queues;
801 i = curcpu % adapter->num_queues;
803 txr = &adapter->tx_rings[i];
804 que = &adapter->queues[i];
806 err = drbr_enqueue(ifp, txr->br, m);
809 if (IXGBE_TX_TRYLOCK(txr)) {
810 err = ixgbe_mq_start_locked(ifp, txr);
811 IXGBE_TX_UNLOCK(txr);
813 taskqueue_enqueue(que->tq, &txr->txq_task);
819 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
821 struct adapter *adapter = txr->adapter;
823 int enqueued = 0, err = 0;
825 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
826 adapter->link_active == 0)
829 /* Process the queue */
830 #if __FreeBSD_version < 901504
831 next = drbr_dequeue(ifp, txr->br);
832 while (next != NULL) {
833 if ((err = ixgbe_xmit(txr, &next)) != 0) {
835 err = drbr_enqueue(ifp, txr->br, next);
837 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
838 if ((err = ixgbe_xmit(txr, &next)) != 0) {
840 drbr_advance(ifp, txr->br);
842 drbr_putback(ifp, txr->br, next);
847 #if __FreeBSD_version >= 901504
848 drbr_advance(ifp, txr->br);
851 /* Send a copy of the frame to the BPF listener */
852 ETHER_BPF_MTAP(ifp, next);
853 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
855 #if __FreeBSD_version < 901504
856 next = drbr_dequeue(ifp, txr->br);
861 /* Set watchdog on */
862 txr->queue_status = IXGBE_QUEUE_WORKING;
863 txr->watchdog_time = ticks;
866 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
873 * Called from a taskqueue to drain queued transmit packets.
876 ixgbe_deferred_mq_start(void *arg, int pending)
878 struct tx_ring *txr = arg;
879 struct adapter *adapter = txr->adapter;
880 struct ifnet *ifp = adapter->ifp;
883 if (!drbr_empty(ifp, txr->br))
884 ixgbe_mq_start_locked(ifp, txr);
885 IXGBE_TX_UNLOCK(txr);
889 ** Flush all ring buffers
892 ixgbe_qflush(struct ifnet *ifp)
894 struct adapter *adapter = ifp->if_softc;
895 struct tx_ring *txr = adapter->tx_rings;
898 for (int i = 0; i < adapter->num_queues; i++, txr++) {
900 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
902 IXGBE_TX_UNLOCK(txr);
906 #endif /* IXGBE_LEGACY_TX */
908 /*********************************************************************
911 * ixgbe_ioctl is called when the user wants to configure the
914 * return 0 on success, positive on failure
915 **********************************************************************/
918 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
920 struct adapter *adapter = ifp->if_softc;
921 struct ixgbe_hw *hw = &adapter->hw;
922 struct ifreq *ifr = (struct ifreq *) data;
923 #if defined(INET) || defined(INET6)
924 struct ifaddr *ifa = (struct ifaddr *)data;
925 bool avoid_reset = FALSE;
933 if (ifa->ifa_addr->sa_family == AF_INET)
937 if (ifa->ifa_addr->sa_family == AF_INET6)
940 #if defined(INET) || defined(INET6)
942 ** Calling init results in link renegotiation,
943 ** so we avoid doing it when possible.
946 ifp->if_flags |= IFF_UP;
947 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
949 if (!(ifp->if_flags & IFF_NOARP))
950 arp_ifinit(ifp, ifa);
952 error = ether_ioctl(ifp, command, data);
956 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
957 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
960 IXGBE_CORE_LOCK(adapter);
961 ifp->if_mtu = ifr->ifr_mtu;
962 adapter->max_frame_size =
963 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
964 ixgbe_init_locked(adapter);
965 IXGBE_CORE_UNLOCK(adapter);
969 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
970 IXGBE_CORE_LOCK(adapter);
971 if (ifp->if_flags & IFF_UP) {
972 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
973 if ((ifp->if_flags ^ adapter->if_flags) &
974 (IFF_PROMISC | IFF_ALLMULTI)) {
975 ixgbe_set_promisc(adapter);
978 ixgbe_init_locked(adapter);
980 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
982 adapter->if_flags = ifp->if_flags;
983 IXGBE_CORE_UNLOCK(adapter);
987 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
988 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
989 IXGBE_CORE_LOCK(adapter);
990 ixgbe_disable_intr(adapter);
991 ixgbe_set_multi(adapter);
992 ixgbe_enable_intr(adapter);
993 IXGBE_CORE_UNLOCK(adapter);
998 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
999 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1003 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1004 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1005 if (mask & IFCAP_HWCSUM)
1006 ifp->if_capenable ^= IFCAP_HWCSUM;
1007 if (mask & IFCAP_TSO4)
1008 ifp->if_capenable ^= IFCAP_TSO4;
1009 if (mask & IFCAP_TSO6)
1010 ifp->if_capenable ^= IFCAP_TSO6;
1011 if (mask & IFCAP_LRO)
1012 ifp->if_capenable ^= IFCAP_LRO;
1013 if (mask & IFCAP_VLAN_HWTAGGING)
1014 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1015 if (mask & IFCAP_VLAN_HWFILTER)
1016 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1017 if (mask & IFCAP_VLAN_HWTSO)
1018 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1019 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1020 IXGBE_CORE_LOCK(adapter);
1021 ixgbe_init_locked(adapter);
1022 IXGBE_CORE_UNLOCK(adapter);
1024 VLAN_CAPABILITIES(ifp);
1029 struct ixgbe_i2c_req i2c;
1030 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1031 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1034 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1038 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1039 i2c.dev_addr, i2c.data);
1040 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1044 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1045 error = ether_ioctl(ifp, command, data);
1052 /*********************************************************************
1055 * This routine is used in two ways. It is used by the stack as
1056 * init entry point in network interface structure. It is also used
1057 * by the driver as a hw/sw initialization routine to get to a
1060 * return 0 on success, positive on failure
1061 **********************************************************************/
1062 #define IXGBE_MHADD_MFS_SHIFT 16
1065 ixgbe_init_locked(struct adapter *adapter)
1067 struct ifnet *ifp = adapter->ifp;
1068 device_t dev = adapter->dev;
1069 struct ixgbe_hw *hw = &adapter->hw;
1070 u32 k, txdctl, mhadd, gpie;
1073 mtx_assert(&adapter->core_mtx, MA_OWNED);
1074 INIT_DEBUGOUT("ixgbe_init: begin");
1075 hw->adapter_stopped = FALSE;
1076 ixgbe_stop_adapter(hw);
1077 callout_stop(&adapter->timer);
1079 /* reprogram the RAR[0] in case user changed it. */
1080 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1082 /* Get the latest mac address, User can use a LAA */
1083 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1084 IXGBE_ETH_LENGTH_OF_ADDRESS);
1085 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1086 hw->addr_ctrl.rar_used_count = 1;
1088 /* Set the various hardware offload abilities */
1089 ifp->if_hwassist = 0;
1090 if (ifp->if_capenable & IFCAP_TSO)
1091 ifp->if_hwassist |= CSUM_TSO;
1092 if (ifp->if_capenable & IFCAP_TXCSUM) {
1093 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1094 #if __FreeBSD_version >= 800000
1095 if (hw->mac.type != ixgbe_mac_82598EB)
1096 ifp->if_hwassist |= CSUM_SCTP;
1100 /* Prepare transmit descriptors and buffers */
1101 if (ixgbe_setup_transmit_structures(adapter)) {
1102 device_printf(dev,"Could not setup transmit structures\n");
1103 ixgbe_stop(adapter);
1108 ixgbe_initialize_transmit_units(adapter);
1110 /* Setup Multicast table */
1111 ixgbe_set_multi(adapter);
1114 ** Determine the correct mbuf pool
1115 ** for doing jumbo frames
1117 if (adapter->max_frame_size <= 2048)
1118 adapter->rx_mbuf_sz = MCLBYTES;
1119 else if (adapter->max_frame_size <= 4096)
1120 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1121 else if (adapter->max_frame_size <= 9216)
1122 adapter->rx_mbuf_sz = MJUM9BYTES;
1124 adapter->rx_mbuf_sz = MJUM16BYTES;
1126 /* Prepare receive descriptors and buffers */
1127 if (ixgbe_setup_receive_structures(adapter)) {
1128 device_printf(dev,"Could not setup receive structures\n");
1129 ixgbe_stop(adapter);
1133 /* Configure RX settings */
1134 ixgbe_initialize_receive_units(adapter);
1136 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1138 /* Enable Fan Failure Interrupt */
1139 gpie |= IXGBE_SDP1_GPIEN;
1141 /* Add for Module detection */
1142 if (hw->mac.type == ixgbe_mac_82599EB)
1143 gpie |= IXGBE_SDP2_GPIEN;
1145 /* Thermal Failure Detection */
1146 if (hw->mac.type == ixgbe_mac_X540)
1147 gpie |= IXGBE_SDP0_GPIEN;
1149 if (adapter->msix > 1) {
1150 /* Enable Enhanced MSIX mode */
1151 gpie |= IXGBE_GPIE_MSIX_MODE;
1152 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1155 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1158 if (ifp->if_mtu > ETHERMTU) {
1159 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1160 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1161 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1162 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1165 /* Now enable all the queues */
1167 for (int i = 0; i < adapter->num_queues; i++) {
1168 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1169 txdctl |= IXGBE_TXDCTL_ENABLE;
1170 /* Set WTHRESH to 8, burst writeback */
1171 txdctl |= (8 << 16);
1173 * When the internal queue falls below PTHRESH (32),
1174 * start prefetching as long as there are at least
1175 * HTHRESH (1) buffers ready. The values are taken
1176 * from the Intel linux driver 3.8.21.
1177 * Prefetching enables tx line rate even with 1 queue.
1179 txdctl |= (32 << 0) | (1 << 8);
1180 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1183 for (int i = 0; i < adapter->num_queues; i++) {
1184 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1185 if (hw->mac.type == ixgbe_mac_82598EB) {
1191 rxdctl &= ~0x3FFFFF;
1194 rxdctl |= IXGBE_RXDCTL_ENABLE;
1195 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1196 for (k = 0; k < 10; k++) {
1197 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1198 IXGBE_RXDCTL_ENABLE)
1206 * In netmap mode, we must preserve the buffers made
1207 * available to userspace before the if_init()
1208 * (this is true by default on the TX side, because
1209 * init makes all buffers available to userspace).
1211 * netmap_reset() and the device specific routines
1212 * (e.g. ixgbe_setup_receive_rings()) map these
1213 * buffers at the end of the NIC ring, so here we
1214 * must set the RDT (tail) register to make sure
1215 * they are not overwritten.
1217 * In this driver the NIC ring starts at RDH = 0,
1218 * RDT points to the last slot available for reception (?),
1219 * so RDT = num_rx_desc - 1 means the whole ring is available.
1221 if (ifp->if_capenable & IFCAP_NETMAP) {
1222 struct netmap_adapter *na = NA(adapter->ifp);
1223 struct netmap_kring *kring = &na->rx_rings[i];
1224 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1226 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1228 #endif /* DEV_NETMAP */
1229 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1232 /* Set up VLAN support and filter */
1233 ixgbe_setup_vlan_hw_support(adapter);
1235 /* Enable Receive engine */
1236 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1237 if (hw->mac.type == ixgbe_mac_82598EB)
1238 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1239 rxctrl |= IXGBE_RXCTRL_RXEN;
1240 ixgbe_enable_rx_dma(hw, rxctrl);
1242 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1244 /* Set up MSI/X routing */
1245 if (ixgbe_enable_msix) {
1246 ixgbe_configure_ivars(adapter);
1247 /* Set up auto-mask */
1248 if (hw->mac.type == ixgbe_mac_82598EB)
1249 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1251 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1252 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1254 } else { /* Simple settings for Legacy/MSI */
1255 ixgbe_set_ivar(adapter, 0, 0, 0);
1256 ixgbe_set_ivar(adapter, 0, 0, 1);
1257 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1261 /* Init Flow director */
1262 if (hw->mac.type != ixgbe_mac_82598EB) {
1263 u32 hdrm = 32 << fdir_pballoc;
1265 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1266 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1271 ** Check on any SFP devices that
1272 ** need to be kick-started
1274 if (hw->phy.type == ixgbe_phy_none) {
1275 int err = hw->phy.ops.identify(hw);
1276 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1278 "Unsupported SFP+ module type was detected.\n");
1283 /* Set moderation on the Link interrupt */
1284 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1286 /* Config/Enable Link */
1287 ixgbe_config_link(adapter);
1289 /* Hardware Packet Buffer & Flow Control setup */
1291 u32 rxpb, frame, size, tmp;
1293 frame = adapter->max_frame_size;
1295 /* Calculate High Water */
1296 if (hw->mac.type == ixgbe_mac_X540)
1297 tmp = IXGBE_DV_X540(frame, frame);
1299 tmp = IXGBE_DV(frame, frame);
1300 size = IXGBE_BT2KB(tmp);
1301 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1302 hw->fc.high_water[0] = rxpb - size;
1304 /* Now calculate Low Water */
1305 if (hw->mac.type == ixgbe_mac_X540)
1306 tmp = IXGBE_LOW_DV_X540(frame);
1308 tmp = IXGBE_LOW_DV(frame);
1309 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1311 hw->fc.requested_mode = adapter->fc;
1312 hw->fc.pause_time = IXGBE_FC_PAUSE;
1313 hw->fc.send_xon = TRUE;
1315 /* Initialize the FC settings */
1318 /* And now turn on interrupts */
1319 ixgbe_enable_intr(adapter);
1321 /* Now inform the stack we're ready */
1322 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1328 ixgbe_init(void *arg)
1330 struct adapter *adapter = arg;
1332 IXGBE_CORE_LOCK(adapter);
1333 ixgbe_init_locked(adapter);
1334 IXGBE_CORE_UNLOCK(adapter);
1341 ** MSIX Interrupt Handlers and Tasklets
1346 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1348 struct ixgbe_hw *hw = &adapter->hw;
1349 u64 queue = (u64)(1 << vector);
1352 if (hw->mac.type == ixgbe_mac_82598EB) {
1353 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1354 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1356 mask = (queue & 0xFFFFFFFF);
1358 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1359 mask = (queue >> 32);
1361 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1366 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1368 struct ixgbe_hw *hw = &adapter->hw;
1369 u64 queue = (u64)(1 << vector);
1372 if (hw->mac.type == ixgbe_mac_82598EB) {
1373 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1374 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1376 mask = (queue & 0xFFFFFFFF);
1378 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1379 mask = (queue >> 32);
1381 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1386 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1390 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1391 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1392 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1394 mask = (queues & 0xFFFFFFFF);
1395 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1396 mask = (queues >> 32);
1397 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1403 ixgbe_handle_que(void *context, int pending)
1405 struct ix_queue *que = context;
1406 struct adapter *adapter = que->adapter;
1407 struct tx_ring *txr = que->txr;
1408 struct ifnet *ifp = adapter->ifp;
1411 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1412 more = ixgbe_rxeof(que);
1415 #ifndef IXGBE_LEGACY_TX
1416 if (!drbr_empty(ifp, txr->br))
1417 ixgbe_mq_start_locked(ifp, txr);
1419 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1420 ixgbe_start_locked(txr, ifp);
1422 IXGBE_TX_UNLOCK(txr);
1425 /* Reenable this interrupt */
1426 if (que->res != NULL)
1427 ixgbe_enable_queue(adapter, que->msix);
1429 ixgbe_enable_intr(adapter);
1434 /*********************************************************************
1436 * Legacy Interrupt Service routine
1438 **********************************************************************/
1441 ixgbe_legacy_irq(void *arg)
1443 struct ix_queue *que = arg;
1444 struct adapter *adapter = que->adapter;
1445 struct ixgbe_hw *hw = &adapter->hw;
1446 struct ifnet *ifp = adapter->ifp;
1447 struct tx_ring *txr = adapter->tx_rings;
1452 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1455 if (reg_eicr == 0) {
1456 ixgbe_enable_intr(adapter);
1460 more = ixgbe_rxeof(que);
1464 #ifdef IXGBE_LEGACY_TX
1465 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1466 ixgbe_start_locked(txr, ifp);
1468 if (!drbr_empty(ifp, txr->br))
1469 ixgbe_mq_start_locked(ifp, txr);
1471 IXGBE_TX_UNLOCK(txr);
1473 /* Check for fan failure */
1474 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1475 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1476 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1477 "REPLACE IMMEDIATELY!!\n");
1478 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1481 /* Link status change */
1482 if (reg_eicr & IXGBE_EICR_LSC)
1483 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1486 taskqueue_enqueue(que->tq, &que->que_task);
1488 ixgbe_enable_intr(adapter);
1493 /*********************************************************************
1495 * MSIX Queue Interrupt Service routine
1497 **********************************************************************/
1499 ixgbe_msix_que(void *arg)
1501 struct ix_queue *que = arg;
1502 struct adapter *adapter = que->adapter;
1503 struct ifnet *ifp = adapter->ifp;
1504 struct tx_ring *txr = que->txr;
1505 struct rx_ring *rxr = que->rxr;
1509 ixgbe_disable_queue(adapter, que->msix);
1512 more = ixgbe_rxeof(que);
1516 #ifdef IXGBE_LEGACY_TX
1517 if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1518 ixgbe_start_locked(txr, ifp);
1520 if (!drbr_empty(ifp, txr->br))
1521 ixgbe_mq_start_locked(ifp, txr);
1523 IXGBE_TX_UNLOCK(txr);
1527 if (ixgbe_enable_aim == FALSE)
1530 ** Do Adaptive Interrupt Moderation:
1531 ** - Write out last calculated setting
1532 ** - Calculate based on average size over
1533 ** the last interval.
1535 if (que->eitr_setting)
1536 IXGBE_WRITE_REG(&adapter->hw,
1537 IXGBE_EITR(que->msix), que->eitr_setting);
1539 que->eitr_setting = 0;
1541 /* Idle, do nothing */
1542 if ((txr->bytes == 0) && (rxr->bytes == 0))
1545 if ((txr->bytes) && (txr->packets))
1546 newitr = txr->bytes/txr->packets;
1547 if ((rxr->bytes) && (rxr->packets))
1548 newitr = max(newitr,
1549 (rxr->bytes / rxr->packets));
1550 newitr += 24; /* account for hardware frame, crc */
1552 /* set an upper boundary */
1553 newitr = min(newitr, 3000);
1555 /* Be nice to the mid range */
1556 if ((newitr > 300) && (newitr < 1200))
1557 newitr = (newitr / 3);
1559 newitr = (newitr / 2);
1561 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1562 newitr |= newitr << 16;
1564 newitr |= IXGBE_EITR_CNT_WDIS;
1566 /* save for next interrupt */
1567 que->eitr_setting = newitr;
1577 taskqueue_enqueue(que->tq, &que->que_task);
1579 ixgbe_enable_queue(adapter, que->msix);
1585 ixgbe_msix_link(void *arg)
1587 struct adapter *adapter = arg;
1588 struct ixgbe_hw *hw = &adapter->hw;
1591 ++adapter->link_irq;
1593 /* First get the cause */
1594 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1595 /* Clear interrupt with write */
1596 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1598 /* Link status change */
1599 if (reg_eicr & IXGBE_EICR_LSC)
1600 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1602 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1604 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1605 /* This is probably overkill :) */
1606 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1608 /* Disable the interrupt */
1609 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1610 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1613 if (reg_eicr & IXGBE_EICR_ECC) {
1614 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1615 "Please Reboot!!\n");
1616 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1619 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1620 /* Clear the interrupt */
1621 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1622 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1623 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1624 /* Clear the interrupt */
1625 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1626 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1630 /* Check for fan failure */
1631 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1632 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1633 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1634 "REPLACE IMMEDIATELY!!\n");
1635 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1638 /* Check for over temp condition */
1639 if ((hw->mac.type == ixgbe_mac_X540) &&
1640 (reg_eicr & IXGBE_EICR_TS)) {
1641 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1642 "PHY IS SHUT DOWN!!\n");
1643 device_printf(adapter->dev, "System shutdown required\n");
1644 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1647 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1651 /*********************************************************************
1653 * Media Ioctl callback
1655 * This routine is called whenever the user queries the status of
1656 * the interface using ifconfig.
1658 **********************************************************************/
1660 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1662 struct adapter *adapter = ifp->if_softc;
1664 INIT_DEBUGOUT("ixgbe_media_status: begin");
1665 IXGBE_CORE_LOCK(adapter);
1666 ixgbe_update_link_status(adapter);
1668 ifmr->ifm_status = IFM_AVALID;
1669 ifmr->ifm_active = IFM_ETHER;
1671 if (!adapter->link_active) {
1672 IXGBE_CORE_UNLOCK(adapter);
1676 ifmr->ifm_status |= IFM_ACTIVE;
1678 switch (adapter->link_speed) {
1679 case IXGBE_LINK_SPEED_100_FULL:
1680 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1682 case IXGBE_LINK_SPEED_1GB_FULL:
1683 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1685 case IXGBE_LINK_SPEED_10GB_FULL:
1686 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1690 IXGBE_CORE_UNLOCK(adapter);
1695 /*********************************************************************
1697 * Media Ioctl callback
1699 * This routine is called when the user changes speed/duplex using
1700 * media/mediopt option with ifconfig.
1702 **********************************************************************/
1704 ixgbe_media_change(struct ifnet * ifp)
1706 struct adapter *adapter = ifp->if_softc;
1707 struct ifmedia *ifm = &adapter->media;
1709 INIT_DEBUGOUT("ixgbe_media_change: begin");
1711 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1714 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1716 adapter->hw.phy.autoneg_advertised =
1717 IXGBE_LINK_SPEED_100_FULL |
1718 IXGBE_LINK_SPEED_1GB_FULL |
1719 IXGBE_LINK_SPEED_10GB_FULL;
1722 device_printf(adapter->dev, "Only auto media type\n");
1729 /*********************************************************************
1731 * This routine maps the mbufs to tx descriptors, allowing the
1732 * TX engine to transmit the packets.
1733 * - return 0 on success, positive on failure
1735 **********************************************************************/
1738 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1740 struct adapter *adapter = txr->adapter;
1741 u32 olinfo_status = 0, cmd_type_len;
1742 int i, j, error, nsegs;
1745 struct mbuf *m_head;
1746 bus_dma_segment_t segs[adapter->num_segs];
1748 struct ixgbe_tx_buf *txbuf;
1749 union ixgbe_adv_tx_desc *txd = NULL;
1753 /* Basic descriptor defines */
1754 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1755 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1757 if (m_head->m_flags & M_VLANTAG)
1758 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1761 * Important to capture the first descriptor
1762 * used because it will contain the index of
1763 * the one we tell the hardware to report back
1765 first = txr->next_avail_desc;
1766 txbuf = &txr->tx_buffers[first];
1770 * Map the packet for DMA.
1773 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1774 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1776 if (__predict_false(error)) {
1781 /* Try it again? - one try */
1782 if (remap == TRUE) {
1784 m = m_defrag(*m_headp, M_NOWAIT);
1786 adapter->mbuf_defrag_failed++;
1796 txr->no_tx_dma_setup++;
1799 txr->no_tx_dma_setup++;
1806 /* Make certain there are enough descriptors */
1807 if (nsegs > txr->tx_avail - 2) {
1808 txr->no_desc_avail++;
1809 bus_dmamap_unload(txr->txtag, map);
1815 ** Set up the appropriate offload context
1816 ** this will consume the first descriptor
1818 error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1819 if (__predict_false(error)) {
1820 if (error == ENOBUFS)
1826 /* Do the flow director magic */
1827 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1829 if (txr->atr_count >= atr_sample_rate) {
1830 ixgbe_atr(txr, m_head);
1836 i = txr->next_avail_desc;
1837 for (j = 0; j < nsegs; j++) {
1841 txbuf = &txr->tx_buffers[i];
1842 txd = &txr->tx_base[i];
1843 seglen = segs[j].ds_len;
1844 segaddr = htole64(segs[j].ds_addr);
1846 txd->read.buffer_addr = segaddr;
1847 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1848 cmd_type_len |seglen);
1849 txd->read.olinfo_status = htole32(olinfo_status);
1851 if (++i == txr->num_desc)
1855 txd->read.cmd_type_len |=
1856 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1857 txr->tx_avail -= nsegs;
1858 txr->next_avail_desc = i;
1860 txbuf->m_head = m_head;
1862 ** Here we swap the map so the last descriptor,
1863 ** which gets the completion interrupt has the
1864 ** real map, and the first descriptor gets the
1865 ** unused map from this descriptor.
1867 txr->tx_buffers[first].map = txbuf->map;
1869 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1871 /* Set the EOP descriptor that will be marked done */
1872 txbuf = &txr->tx_buffers[first];
1875 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1876 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1878 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1879 * hardware that this frame is available to transmit.
1881 ++txr->total_packets;
1882 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1889 ixgbe_set_promisc(struct adapter *adapter)
1892 struct ifnet *ifp = adapter->ifp;
1895 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1896 reg_rctl &= (~IXGBE_FCTRL_UPE);
1897 if (ifp->if_flags & IFF_ALLMULTI)
1898 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1900 struct ifmultiaddr *ifma;
1901 #if __FreeBSD_version < 800000
1904 if_maddr_rlock(ifp);
1906 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1907 if (ifma->ifma_addr->sa_family != AF_LINK)
1909 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1913 #if __FreeBSD_version < 800000
1914 IF_ADDR_UNLOCK(ifp);
1916 if_maddr_runlock(ifp);
1919 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1920 reg_rctl &= (~IXGBE_FCTRL_MPE);
1921 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1923 if (ifp->if_flags & IFF_PROMISC) {
1924 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1925 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1926 } else if (ifp->if_flags & IFF_ALLMULTI) {
1927 reg_rctl |= IXGBE_FCTRL_MPE;
1928 reg_rctl &= ~IXGBE_FCTRL_UPE;
1929 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1935 /*********************************************************************
1938 * This routine is called whenever multicast address list is updated.
1940 **********************************************************************/
1941 #define IXGBE_RAR_ENTRIES 16
1944 ixgbe_set_multi(struct adapter *adapter)
1949 struct ifmultiaddr *ifma;
1951 struct ifnet *ifp = adapter->ifp;
1953 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1956 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1957 MAX_NUM_MULTICAST_ADDRESSES);
1959 #if __FreeBSD_version < 800000
1962 if_maddr_rlock(ifp);
1964 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1965 if (ifma->ifma_addr->sa_family != AF_LINK)
1967 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1969 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1970 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1971 IXGBE_ETH_LENGTH_OF_ADDRESS);
1974 #if __FreeBSD_version < 800000
1975 IF_ADDR_UNLOCK(ifp);
1977 if_maddr_runlock(ifp);
1980 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1981 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1982 if (ifp->if_flags & IFF_PROMISC)
1983 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1984 else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1985 ifp->if_flags & IFF_ALLMULTI) {
1986 fctrl |= IXGBE_FCTRL_MPE;
1987 fctrl &= ~IXGBE_FCTRL_UPE;
1989 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1991 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1993 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
1995 ixgbe_update_mc_addr_list(&adapter->hw,
1996 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
2003 * This is an iterator function now needed by the multicast
2004 * shared code. It simply feeds the shared code routine the
2005 * addresses in the array of ixgbe_set_multi() one by one.
2008 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2010 u8 *addr = *update_ptr;
2014 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2015 *update_ptr = newptr;
2020 /*********************************************************************
2023 * This routine checks for link status,updates statistics,
2024 * and runs the watchdog check.
2026 **********************************************************************/
2029 ixgbe_local_timer(void *arg)
2031 struct adapter *adapter = arg;
2032 device_t dev = adapter->dev;
2033 struct ix_queue *que = adapter->queues;
2034 struct tx_ring *txr = adapter->tx_rings;
2035 int hung = 0, paused = 0;
2037 mtx_assert(&adapter->core_mtx, MA_OWNED);
2039 /* Check for pluggable optics */
2040 if (adapter->sfp_probe)
2041 if (!ixgbe_sfp_probe(adapter))
2042 goto out; /* Nothing to do */
2044 ixgbe_update_link_status(adapter);
2045 ixgbe_update_stats_counters(adapter);
2048 * If the interface has been paused
2049 * then don't do the watchdog check
2051 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2055 ** Check the TX queues status
2056 ** - watchdog only if all queues show hung
2058 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2059 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2062 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2063 taskqueue_enqueue(que->tq, &txr->txq_task);
2065 /* Only truely watchdog if all queues show hung */
2066 if (hung == adapter->num_queues)
2070 ixgbe_rearm_queues(adapter, adapter->que_mask);
2071 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2075 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2076 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2077 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2078 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2079 device_printf(dev,"TX(%d) desc avail = %d,"
2080 "Next TX to Clean = %d\n",
2081 txr->me, txr->tx_avail, txr->next_to_clean);
2082 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2083 adapter->watchdog_events++;
2084 ixgbe_init_locked(adapter);
2088 ** Note: this routine updates the OS on the link state
2089 ** the real check of the hardware only happens with
2090 ** a link interrupt.
2093 ixgbe_update_link_status(struct adapter *adapter)
2095 struct ifnet *ifp = adapter->ifp;
2096 device_t dev = adapter->dev;
2099 if (adapter->link_up){
2100 if (adapter->link_active == FALSE) {
2102 device_printf(dev,"Link is up %d Gbps %s \n",
2103 ((adapter->link_speed == 128)? 10:1),
2105 adapter->link_active = TRUE;
2106 /* Update any Flow Control changes */
2107 ixgbe_fc_enable(&adapter->hw);
2108 if_link_state_change(ifp, LINK_STATE_UP);
2110 } else { /* Link down */
2111 if (adapter->link_active == TRUE) {
2113 device_printf(dev,"Link is Down\n");
2114 if_link_state_change(ifp, LINK_STATE_DOWN);
2115 adapter->link_active = FALSE;
2123 /*********************************************************************
2125 * This routine disables all traffic on the adapter by issuing a
2126 * global reset on the MAC and deallocates TX/RX buffers.
2128 **********************************************************************/
2131 ixgbe_stop(void *arg)
2134 struct adapter *adapter = arg;
2135 struct ixgbe_hw *hw = &adapter->hw;
2138 mtx_assert(&adapter->core_mtx, MA_OWNED);
2140 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2141 ixgbe_disable_intr(adapter);
2142 callout_stop(&adapter->timer);
2144 /* Let the stack know...*/
2145 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2148 hw->adapter_stopped = FALSE;
2149 ixgbe_stop_adapter(hw);
2150 if (hw->mac.type == ixgbe_mac_82599EB)
2151 ixgbe_stop_mac_link_on_d3_82599(hw);
2152 /* Turn off the laser - noop with no optics */
2153 ixgbe_disable_tx_laser(hw);
2155 /* Update the stack */
2156 adapter->link_up = FALSE;
2157 ixgbe_update_link_status(adapter);
2159 /* reprogram the RAR[0] in case user changed it. */
2160 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2166 /*********************************************************************
2168 * Determine hardware revision.
2170 **********************************************************************/
2172 ixgbe_identify_hardware(struct adapter *adapter)
2174 device_t dev = adapter->dev;
2175 struct ixgbe_hw *hw = &adapter->hw;
2177 /* Save off the information about this board */
2178 hw->vendor_id = pci_get_vendor(dev);
2179 hw->device_id = pci_get_device(dev);
2180 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2181 hw->subsystem_vendor_id =
2182 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2183 hw->subsystem_device_id =
2184 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2186 /* We need this here to set the num_segs below */
2187 ixgbe_set_mac_type(hw);
2189 /* Pick up the 82599 and VF settings */
2190 if (hw->mac.type != ixgbe_mac_82598EB) {
2191 hw->phy.smart_speed = ixgbe_smart_speed;
2192 adapter->num_segs = IXGBE_82599_SCATTER;
2194 adapter->num_segs = IXGBE_82598_SCATTER;
2199 /*********************************************************************
2201 * Determine optic type
2203 **********************************************************************/
2205 ixgbe_setup_optics(struct adapter *adapter)
2207 struct ixgbe_hw *hw = &adapter->hw;
2210 layer = ixgbe_get_supported_physical_layer(hw);
2212 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2213 adapter->optics = IFM_10G_T;
2217 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2218 adapter->optics = IFM_1000_T;
2222 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2223 adapter->optics = IFM_1000_SX;
2227 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2228 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2229 adapter->optics = IFM_10G_LR;
2233 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2234 adapter->optics = IFM_10G_SR;
2238 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2239 adapter->optics = IFM_10G_TWINAX;
2243 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2244 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2245 adapter->optics = IFM_10G_CX4;
2249 /* If we get here just set the default */
2250 adapter->optics = IFM_ETHER | IFM_AUTO;
2254 /*********************************************************************
2256 * Setup the Legacy or MSI Interrupt handler
2258 **********************************************************************/
2260 ixgbe_allocate_legacy(struct adapter *adapter)
2262 device_t dev = adapter->dev;
2263 struct ix_queue *que = adapter->queues;
2264 #ifndef IXGBE_LEGACY_TX
2265 struct tx_ring *txr = adapter->tx_rings;
2270 if (adapter->msix == 1)
2273 /* We allocate a single interrupt resource */
2274 adapter->res = bus_alloc_resource_any(dev,
2275 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2276 if (adapter->res == NULL) {
2277 device_printf(dev, "Unable to allocate bus resource: "
2283 * Try allocating a fast interrupt and the associated deferred
2284 * processing contexts.
2286 #ifndef IXGBE_LEGACY_TX
2287 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2289 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2290 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2291 taskqueue_thread_enqueue, &que->tq);
2292 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2293 device_get_nameunit(adapter->dev));
2295 /* Tasklets for Link, SFP and Multispeed Fiber */
2296 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2297 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2298 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2300 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2302 adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2303 taskqueue_thread_enqueue, &adapter->tq);
2304 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2305 device_get_nameunit(adapter->dev));
2307 if ((error = bus_setup_intr(dev, adapter->res,
2308 INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2309 que, &adapter->tag)) != 0) {
2310 device_printf(dev, "Failed to register fast interrupt "
2311 "handler: %d\n", error);
2312 taskqueue_free(que->tq);
2313 taskqueue_free(adapter->tq);
2318 /* For simplicity in the handlers */
2319 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2325 /*********************************************************************
2327 * Setup MSIX Interrupt resources and handlers
2329 **********************************************************************/
2331 ixgbe_allocate_msix(struct adapter *adapter)
2333 device_t dev = adapter->dev;
2334 struct ix_queue *que = adapter->queues;
2335 struct tx_ring *txr = adapter->tx_rings;
2336 int error, rid, vector = 0;
2338 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2340 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2341 RF_SHAREABLE | RF_ACTIVE);
2342 if (que->res == NULL) {
2343 device_printf(dev,"Unable to allocate"
2344 " bus resource: que interrupt [%d]\n", vector);
2347 /* Set the handler function */
2348 error = bus_setup_intr(dev, que->res,
2349 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2350 ixgbe_msix_que, que, &que->tag);
2353 device_printf(dev, "Failed to register QUE handler");
2356 #if __FreeBSD_version >= 800504
2357 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2360 adapter->que_mask |= (u64)(1 << que->msix);
2362 ** Bind the msix vector, and thus the
2363 ** ring to the corresponding cpu.
2365 if (adapter->num_queues > 1)
2366 bus_bind_intr(dev, que->res, i);
2368 #ifndef IXGBE_LEGACY_TX
2369 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2371 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2372 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2373 taskqueue_thread_enqueue, &que->tq);
2374 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2375 device_get_nameunit(adapter->dev));
2380 adapter->res = bus_alloc_resource_any(dev,
2381 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2382 if (!adapter->res) {
2383 device_printf(dev,"Unable to allocate"
2384 " bus resource: Link interrupt [%d]\n", rid);
2387 /* Set the link handler function */
2388 error = bus_setup_intr(dev, adapter->res,
2389 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2390 ixgbe_msix_link, adapter, &adapter->tag);
2392 adapter->res = NULL;
2393 device_printf(dev, "Failed to register LINK handler");
2396 #if __FreeBSD_version >= 800504
2397 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2399 adapter->linkvec = vector;
2400 /* Tasklets for Link, SFP and Multispeed Fiber */
2401 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2402 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2403 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2405 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2407 adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2408 taskqueue_thread_enqueue, &adapter->tq);
2409 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2410 device_get_nameunit(adapter->dev));
2416 * Setup Either MSI/X or MSI
2419 ixgbe_setup_msix(struct adapter *adapter)
2421 device_t dev = adapter->dev;
2422 int rid, want, queues, msgs;
2424 /* Override by tuneable */
2425 if (ixgbe_enable_msix == 0)
2428 /* First try MSI/X */
2429 rid = PCIR_BAR(MSIX_82598_BAR);
2430 adapter->msix_mem = bus_alloc_resource_any(dev,
2431 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2432 if (!adapter->msix_mem) {
2433 rid += 4; /* 82599 maps in higher BAR */
2434 adapter->msix_mem = bus_alloc_resource_any(dev,
2435 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2437 if (!adapter->msix_mem) {
2438 /* May not be enabled */
2439 device_printf(adapter->dev,
2440 "Unable to map MSIX table \n");
2444 msgs = pci_msix_count(dev);
2445 if (msgs == 0) { /* system has msix disabled */
2446 bus_release_resource(dev, SYS_RES_MEMORY,
2447 rid, adapter->msix_mem);
2448 adapter->msix_mem = NULL;
2452 /* Figure out a reasonable auto config value */
2453 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2455 if (ixgbe_num_queues != 0)
2456 queues = ixgbe_num_queues;
2457 /* Set max queues to 8 when autoconfiguring */
2458 else if ((ixgbe_num_queues == 0) && (queues > 8))
2462 ** Want one vector (RX/TX pair) per queue
2463 ** plus an additional for Link.
2469 device_printf(adapter->dev,
2470 "MSIX Configuration Problem, "
2471 "%d vectors but %d queues wanted!\n",
2473 return (0); /* Will go to Legacy setup */
2475 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2476 device_printf(adapter->dev,
2477 "Using MSIX interrupts with %d vectors\n", msgs);
2478 adapter->num_queues = queues;
2482 msgs = pci_msi_count(dev);
2483 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2484 device_printf(adapter->dev,"Using an MSI interrupt\n");
2486 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2492 ixgbe_allocate_pci_resources(struct adapter *adapter)
2495 device_t dev = adapter->dev;
2498 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2501 if (!(adapter->pci_mem)) {
2502 device_printf(dev,"Unable to allocate bus resource: memory\n");
2506 adapter->osdep.mem_bus_space_tag =
2507 rman_get_bustag(adapter->pci_mem);
2508 adapter->osdep.mem_bus_space_handle =
2509 rman_get_bushandle(adapter->pci_mem);
2510 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2512 /* Legacy defaults */
2513 adapter->num_queues = 1;
2514 adapter->hw.back = &adapter->osdep;
2517 ** Now setup MSI or MSI/X, should
2518 ** return us the number of supported
2519 ** vectors. (Will be 1 for MSI)
2521 adapter->msix = ixgbe_setup_msix(adapter);
2526 ixgbe_free_pci_resources(struct adapter * adapter)
2528 struct ix_queue *que = adapter->queues;
2529 device_t dev = adapter->dev;
2532 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2533 memrid = PCIR_BAR(MSIX_82598_BAR);
2535 memrid = PCIR_BAR(MSIX_82599_BAR);
2538 ** There is a slight possibility of a failure mode
2539 ** in attach that will result in entering this function
2540 ** before interrupt resources have been initialized, and
2541 ** in that case we do not want to execute the loops below
2542 ** We can detect this reliably by the state of the adapter
2545 if (adapter->res == NULL)
2549 ** Release all msix queue resources:
2551 for (int i = 0; i < adapter->num_queues; i++, que++) {
2552 rid = que->msix + 1;
2553 if (que->tag != NULL) {
2554 bus_teardown_intr(dev, que->res, que->tag);
2557 if (que->res != NULL)
2558 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2562 /* Clean the Legacy or Link interrupt last */
2563 if (adapter->linkvec) /* we are doing MSIX */
2564 rid = adapter->linkvec + 1;
2566 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2568 if (adapter->tag != NULL) {
2569 bus_teardown_intr(dev, adapter->res, adapter->tag);
2570 adapter->tag = NULL;
2572 if (adapter->res != NULL)
2573 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2577 pci_release_msi(dev);
2579 if (adapter->msix_mem != NULL)
2580 bus_release_resource(dev, SYS_RES_MEMORY,
2581 memrid, adapter->msix_mem);
2583 if (adapter->pci_mem != NULL)
2584 bus_release_resource(dev, SYS_RES_MEMORY,
2585 PCIR_BAR(0), adapter->pci_mem);
2590 /*********************************************************************
2592 * Setup networking device structure and register an interface.
2594 **********************************************************************/
2596 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2598 struct ixgbe_hw *hw = &adapter->hw;
2601 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2603 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2605 device_printf(dev, "can not allocate ifnet structure\n");
2608 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2609 #if __FreeBSD_version < 1000025
2610 ifp->if_baudrate = 1000000000;
2612 if_initbaudrate(ifp, IF_Gbps(10));
2614 ifp->if_init = ixgbe_init;
2615 ifp->if_softc = adapter;
2616 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2617 ifp->if_ioctl = ixgbe_ioctl;
2618 #ifndef IXGBE_LEGACY_TX
2619 ifp->if_transmit = ixgbe_mq_start;
2620 ifp->if_qflush = ixgbe_qflush;
2622 ifp->if_start = ixgbe_start;
2623 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2624 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2625 IFQ_SET_READY(&ifp->if_snd);
2628 ether_ifattach(ifp, adapter->hw.mac.addr);
2630 adapter->max_frame_size =
2631 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2634 * Tell the upper layer(s) we support long frames.
2636 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2638 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2639 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2640 ifp->if_capabilities |= IFCAP_LRO;
2641 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2644 ifp->if_capenable = ifp->if_capabilities;
2647 ** Don't turn this on by default, if vlans are
2648 ** created on another pseudo device (eg. lagg)
2649 ** then vlan events are not passed thru, breaking
2650 ** operation, but with HW FILTER off it works. If
2651 ** using vlans directly on the ixgbe driver you can
2652 ** enable this and get full hardware tag filtering.
2654 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2657 * Specify the media types supported by this adapter and register
2658 * callbacks to update media and link information
2660 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2661 ixgbe_media_status);
2662 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2663 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2664 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2665 ifmedia_add(&adapter->media,
2666 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2667 ifmedia_add(&adapter->media,
2668 IFM_ETHER | IFM_1000_T, 0, NULL);
2670 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2671 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2677 ixgbe_config_link(struct adapter *adapter)
2679 struct ixgbe_hw *hw = &adapter->hw;
2680 u32 autoneg, err = 0;
2681 bool sfp, negotiate;
2683 sfp = ixgbe_is_sfp(hw);
2686 if (hw->phy.multispeed_fiber) {
2687 hw->mac.ops.setup_sfp(hw);
2688 ixgbe_enable_tx_laser(hw);
2689 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2691 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2693 if (hw->mac.ops.check_link)
2694 err = ixgbe_check_link(hw, &adapter->link_speed,
2695 &adapter->link_up, FALSE);
2698 autoneg = hw->phy.autoneg_advertised;
2699 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2700 err = hw->mac.ops.get_link_capabilities(hw,
2701 &autoneg, &negotiate);
2704 if (hw->mac.ops.setup_link)
2705 err = hw->mac.ops.setup_link(hw,
2706 autoneg, adapter->link_up);
2712 /********************************************************************
2713 * Manage DMA'able memory.
2714 *******************************************************************/
2716 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2720 *(bus_addr_t *) arg = segs->ds_addr;
2725 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2726 struct ixgbe_dma_alloc *dma, int mapflags)
2728 device_t dev = adapter->dev;
2731 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2732 DBA_ALIGN, 0, /* alignment, bounds */
2733 BUS_SPACE_MAXADDR, /* lowaddr */
2734 BUS_SPACE_MAXADDR, /* highaddr */
2735 NULL, NULL, /* filter, filterarg */
2738 size, /* maxsegsize */
2739 BUS_DMA_ALLOCNOW, /* flags */
2740 NULL, /* lockfunc */
2741 NULL, /* lockfuncarg */
2744 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2748 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2749 BUS_DMA_NOWAIT, &dma->dma_map);
2751 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2755 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2759 mapflags | BUS_DMA_NOWAIT);
2761 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2765 dma->dma_size = size;
2768 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2770 bus_dma_tag_destroy(dma->dma_tag);
2772 dma->dma_map = NULL;
2773 dma->dma_tag = NULL;
2778 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2780 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2781 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2782 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2783 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2784 bus_dma_tag_destroy(dma->dma_tag);
2788 /*********************************************************************
2790 * Allocate memory for the transmit and receive rings, and then
2791 * the descriptors associated with each, called only once at attach.
2793 **********************************************************************/
2795 ixgbe_allocate_queues(struct adapter *adapter)
2797 device_t dev = adapter->dev;
2798 struct ix_queue *que;
2799 struct tx_ring *txr;
2800 struct rx_ring *rxr;
2801 int rsize, tsize, error = IXGBE_SUCCESS;
2802 int txconf = 0, rxconf = 0;
2804 /* First allocate the top level queue structs */
2805 if (!(adapter->queues =
2806 (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2807 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2808 device_printf(dev, "Unable to allocate queue memory\n");
2813 /* First allocate the TX ring struct memory */
2814 if (!(adapter->tx_rings =
2815 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2816 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2817 device_printf(dev, "Unable to allocate TX ring memory\n");
2822 /* Next allocate the RX */
2823 if (!(adapter->rx_rings =
2824 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2825 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2826 device_printf(dev, "Unable to allocate RX ring memory\n");
2831 /* For the ring itself */
2832 tsize = roundup2(adapter->num_tx_desc *
2833 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2836 * Now set up the TX queues, txconf is needed to handle the
2837 * possibility that things fail midcourse and we need to
2838 * undo memory gracefully
2840 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2841 /* Set up some basics */
2842 txr = &adapter->tx_rings[i];
2843 txr->adapter = adapter;
2845 txr->num_desc = adapter->num_tx_desc;
2847 /* Initialize the TX side lock */
2848 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2849 device_get_nameunit(dev), txr->me);
2850 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2852 if (ixgbe_dma_malloc(adapter, tsize,
2853 &txr->txdma, BUS_DMA_NOWAIT)) {
2855 "Unable to allocate TX Descriptor memory\n");
2859 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2860 bzero((void *)txr->tx_base, tsize);
2862 /* Now allocate transmit buffers for the ring */
2863 if (ixgbe_allocate_transmit_buffers(txr)) {
2865 "Critical Failure setting up transmit buffers\n");
2869 #ifndef IXGBE_LEGACY_TX
2870 /* Allocate a buf ring */
2871 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2872 M_WAITOK, &txr->tx_mtx);
2873 if (txr->br == NULL) {
2875 "Critical Failure setting up buf ring\n");
2883 * Next the RX queues...
2885 rsize = roundup2(adapter->num_rx_desc *
2886 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2887 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2888 rxr = &adapter->rx_rings[i];
2889 /* Set up some basics */
2890 rxr->adapter = adapter;
2892 rxr->num_desc = adapter->num_rx_desc;
2894 /* Initialize the RX side lock */
2895 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2896 device_get_nameunit(dev), rxr->me);
2897 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2899 if (ixgbe_dma_malloc(adapter, rsize,
2900 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2902 "Unable to allocate RxDescriptor memory\n");
2906 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2907 bzero((void *)rxr->rx_base, rsize);
2909 /* Allocate receive buffers for the ring*/
2910 if (ixgbe_allocate_receive_buffers(rxr)) {
2912 "Critical Failure setting up receive buffers\n");
2919 ** Finally set up the queue holding structs
2921 for (int i = 0; i < adapter->num_queues; i++) {
2922 que = &adapter->queues[i];
2923 que->adapter = adapter;
2924 que->txr = &adapter->tx_rings[i];
2925 que->rxr = &adapter->rx_rings[i];
2931 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2932 ixgbe_dma_free(adapter, &rxr->rxdma);
2934 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2935 ixgbe_dma_free(adapter, &txr->txdma);
2936 free(adapter->rx_rings, M_DEVBUF);
2938 free(adapter->tx_rings, M_DEVBUF);
2940 free(adapter->queues, M_DEVBUF);
2945 /*********************************************************************
2947 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2948 * the information needed to transmit a packet on the wire. This is
2949 * called only once at attach, setup is done every reset.
2951 **********************************************************************/
2953 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2955 struct adapter *adapter = txr->adapter;
2956 device_t dev = adapter->dev;
2957 struct ixgbe_tx_buf *txbuf;
2961 * Setup DMA descriptor areas.
2963 if ((error = bus_dma_tag_create(
2964 bus_get_dma_tag(adapter->dev), /* parent */
2965 1, 0, /* alignment, bounds */
2966 BUS_SPACE_MAXADDR, /* lowaddr */
2967 BUS_SPACE_MAXADDR, /* highaddr */
2968 NULL, NULL, /* filter, filterarg */
2969 IXGBE_TSO_SIZE, /* maxsize */
2970 adapter->num_segs, /* nsegments */
2971 PAGE_SIZE, /* maxsegsize */
2973 NULL, /* lockfunc */
2974 NULL, /* lockfuncarg */
2976 device_printf(dev,"Unable to allocate TX DMA tag\n");
2980 if (!(txr->tx_buffers =
2981 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2982 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2983 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2988 /* Create the descriptor buffer dma maps */
2989 txbuf = txr->tx_buffers;
2990 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2991 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2993 device_printf(dev, "Unable to create TX DMA map\n");
3000 /* We free all, it handles case where we are in the middle */
3001 ixgbe_free_transmit_structures(adapter);
3005 /*********************************************************************
3007 * Initialize a transmit ring.
3009 **********************************************************************/
3011 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3013 struct adapter *adapter = txr->adapter;
3014 struct ixgbe_tx_buf *txbuf;
3017 struct netmap_adapter *na = NA(adapter->ifp);
3018 struct netmap_slot *slot;
3019 #endif /* DEV_NETMAP */
3021 /* Clear the old ring contents */
3025 * (under lock): if in netmap mode, do some consistency
3026 * checks and set slot to entry 0 of the netmap ring.
3028 slot = netmap_reset(na, NR_TX, txr->me, 0);
3029 #endif /* DEV_NETMAP */
3030 bzero((void *)txr->tx_base,
3031 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3033 txr->next_avail_desc = 0;
3034 txr->next_to_clean = 0;
3036 /* Free any existing tx buffers. */
3037 txbuf = txr->tx_buffers;
3038 for (i = 0; i < txr->num_desc; i++, txbuf++) {
3039 if (txbuf->m_head != NULL) {
3040 bus_dmamap_sync(txr->txtag, txbuf->map,
3041 BUS_DMASYNC_POSTWRITE);
3042 bus_dmamap_unload(txr->txtag, txbuf->map);
3043 m_freem(txbuf->m_head);
3044 txbuf->m_head = NULL;
3048 * In netmap mode, set the map for the packet buffer.
3049 * NOTE: Some drivers (not this one) also need to set
3050 * the physical buffer address in the NIC ring.
3051 * Slots in the netmap ring (indexed by "si") are
3052 * kring->nkr_hwofs positions "ahead" wrt the
3053 * corresponding slot in the NIC ring. In some drivers
3054 * (not here) nkr_hwofs can be negative. Function
3055 * netmap_idx_n2k() handles wraparounds properly.
3058 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3059 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3061 #endif /* DEV_NETMAP */
3062 /* Clear the EOP descriptor pointer */
3067 /* Set the rate at which we sample packets */
3068 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3069 txr->atr_sample = atr_sample_rate;
3072 /* Set number of descriptors available */
3073 txr->tx_avail = adapter->num_tx_desc;
3075 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3076 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3077 IXGBE_TX_UNLOCK(txr);
3080 /*********************************************************************
3082 * Initialize all transmit rings.
3084 **********************************************************************/
3086 ixgbe_setup_transmit_structures(struct adapter *adapter)
3088 struct tx_ring *txr = adapter->tx_rings;
3090 for (int i = 0; i < adapter->num_queues; i++, txr++)
3091 ixgbe_setup_transmit_ring(txr);
3096 /*********************************************************************
3098 * Enable transmit unit.
3100 **********************************************************************/
3102 ixgbe_initialize_transmit_units(struct adapter *adapter)
3104 struct tx_ring *txr = adapter->tx_rings;
3105 struct ixgbe_hw *hw = &adapter->hw;
3107 /* Setup the Base and Length of the Tx Descriptor Ring */
3109 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3110 u64 tdba = txr->txdma.dma_paddr;
3113 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3114 (tdba & 0x00000000ffffffffULL));
3115 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3116 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3117 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3119 /* Setup the HW Tx Head and Tail descriptor pointers */
3120 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3121 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3123 /* Setup Transmit Descriptor Cmd Settings */
3124 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3125 txr->queue_status = IXGBE_QUEUE_IDLE;
3127 /* Set the processing limit */
3128 txr->process_limit = ixgbe_tx_process_limit;
3130 /* Disable Head Writeback */
3131 switch (hw->mac.type) {
3132 case ixgbe_mac_82598EB:
3133 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3135 case ixgbe_mac_82599EB:
3136 case ixgbe_mac_X540:
3138 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3141 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3142 switch (hw->mac.type) {
3143 case ixgbe_mac_82598EB:
3144 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3146 case ixgbe_mac_82599EB:
3147 case ixgbe_mac_X540:
3149 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3155 if (hw->mac.type != ixgbe_mac_82598EB) {
3156 u32 dmatxctl, rttdcs;
3157 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3158 dmatxctl |= IXGBE_DMATXCTL_TE;
3159 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3160 /* Disable arbiter to set MTQC */
3161 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3162 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3163 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3164 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3165 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3166 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3172 /*********************************************************************
3174 * Free all transmit rings.
3176 **********************************************************************/
3178 ixgbe_free_transmit_structures(struct adapter *adapter)
3180 struct tx_ring *txr = adapter->tx_rings;
3182 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3184 ixgbe_free_transmit_buffers(txr);
3185 ixgbe_dma_free(adapter, &txr->txdma);
3186 IXGBE_TX_UNLOCK(txr);
3187 IXGBE_TX_LOCK_DESTROY(txr);
3189 free(adapter->tx_rings, M_DEVBUF);
3192 /*********************************************************************
3194 * Free transmit ring related data structures.
3196 **********************************************************************/
3198 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3200 struct adapter *adapter = txr->adapter;
3201 struct ixgbe_tx_buf *tx_buffer;
3204 INIT_DEBUGOUT("free_transmit_ring: begin");
3206 if (txr->tx_buffers == NULL)
3209 tx_buffer = txr->tx_buffers;
3210 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3211 if (tx_buffer->m_head != NULL) {
3212 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3213 BUS_DMASYNC_POSTWRITE);
3214 bus_dmamap_unload(txr->txtag,
3216 m_freem(tx_buffer->m_head);
3217 tx_buffer->m_head = NULL;
3218 if (tx_buffer->map != NULL) {
3219 bus_dmamap_destroy(txr->txtag,
3221 tx_buffer->map = NULL;
3223 } else if (tx_buffer->map != NULL) {
3224 bus_dmamap_unload(txr->txtag,
3226 bus_dmamap_destroy(txr->txtag,
3228 tx_buffer->map = NULL;
3231 #ifdef IXGBE_LEGACY_TX
3232 if (txr->br != NULL)
3233 buf_ring_free(txr->br, M_DEVBUF);
3235 if (txr->tx_buffers != NULL) {
3236 free(txr->tx_buffers, M_DEVBUF);
3237 txr->tx_buffers = NULL;
3239 if (txr->txtag != NULL) {
3240 bus_dma_tag_destroy(txr->txtag);
3246 /*********************************************************************
3248 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
3250 **********************************************************************/
3253 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3254 u32 *cmd_type_len, u32 *olinfo_status)
3256 struct ixgbe_adv_tx_context_desc *TXD;
3257 struct ether_vlan_header *eh;
3259 struct ip6_hdr *ip6;
3260 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3261 int ehdrlen, ip_hlen = 0;
3265 int ctxd = txr->next_avail_desc;
3268 /* First check if TSO is to be used */
3269 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3270 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3272 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3275 /* Indicate the whole packet as payload when not doing TSO */
3276 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3278 /* Now ready a context descriptor */
3279 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3282 ** In advanced descriptors the vlan tag must
3283 ** be placed into the context descriptor. Hence
3284 ** we need to make one even if not doing offloads.
3286 if (mp->m_flags & M_VLANTAG) {
3287 vtag = htole16(mp->m_pkthdr.ether_vtag);
3288 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3289 } else if (offload == FALSE) /* ... no offload to do */
3293 * Determine where frame payload starts.
3294 * Jump over vlan headers if already present,
3295 * helpful for QinQ too.
3297 eh = mtod(mp, struct ether_vlan_header *);
3298 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3299 etype = ntohs(eh->evl_proto);
3300 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3302 etype = ntohs(eh->evl_encap_proto);
3303 ehdrlen = ETHER_HDR_LEN;
3306 /* Set the ether header length */
3307 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3311 ip = (struct ip *)(mp->m_data + ehdrlen);
3312 ip_hlen = ip->ip_hl << 2;
3314 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3316 case ETHERTYPE_IPV6:
3317 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3318 ip_hlen = sizeof(struct ip6_hdr);
3319 /* XXX-BZ this will go badly in case of ext hdrs. */
3320 ipproto = ip6->ip6_nxt;
3321 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3328 vlan_macip_lens |= ip_hlen;
3329 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3333 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3334 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3338 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3339 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3342 #if __FreeBSD_version >= 800000
3344 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3345 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3353 if (offload) /* For the TX descriptor setup */
3354 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3356 /* Now copy bits into descriptor */
3357 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3358 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3359 TXD->seqnum_seed = htole32(0);
3360 TXD->mss_l4len_idx = htole32(0);
3362 /* We've consumed the first desc, adjust counters */
3363 if (++ctxd == txr->num_desc)
3365 txr->next_avail_desc = ctxd;
3371 /**********************************************************************
3373 * Setup work for hardware segmentation offload (TSO) on
3374 * adapters using advanced tx descriptors
3376 **********************************************************************/
3378 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3379 u32 *cmd_type_len, u32 *olinfo_status)
3381 struct ixgbe_adv_tx_context_desc *TXD;
3382 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3383 u32 mss_l4len_idx = 0, paylen;
3384 u16 vtag = 0, eh_type;
3385 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3386 struct ether_vlan_header *eh;
3388 struct ip6_hdr *ip6;
3397 * Determine where frame payload starts.
3398 * Jump over vlan headers if already present
3400 eh = mtod(mp, struct ether_vlan_header *);
3401 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3402 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3403 eh_type = eh->evl_proto;
3405 ehdrlen = ETHER_HDR_LEN;
3406 eh_type = eh->evl_encap_proto;
3409 switch (ntohs(eh_type)) {
3411 case ETHERTYPE_IPV6:
3412 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3413 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3414 if (ip6->ip6_nxt != IPPROTO_TCP)
3416 ip_hlen = sizeof(struct ip6_hdr);
3417 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3418 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3419 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3420 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3425 ip = (struct ip *)(mp->m_data + ehdrlen);
3426 if (ip->ip_p != IPPROTO_TCP)
3429 ip_hlen = ip->ip_hl << 2;
3430 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3431 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3432 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3433 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3434 /* Tell transmit desc to also do IPv4 checksum. */
3435 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3439 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3440 __func__, ntohs(eh_type));
3444 ctxd = txr->next_avail_desc;
3445 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3447 tcp_hlen = th->th_off << 2;
3449 /* This is used in the transmit desc in encap */
3450 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3452 /* VLAN MACLEN IPLEN */
3453 if (mp->m_flags & M_VLANTAG) {
3454 vtag = htole16(mp->m_pkthdr.ether_vtag);
3455 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3458 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3459 vlan_macip_lens |= ip_hlen;
3460 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3462 /* ADV DTYPE TUCMD */
3463 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3464 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3465 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3468 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3469 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3470 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3472 TXD->seqnum_seed = htole32(0);
3474 if (++ctxd == txr->num_desc)
3478 txr->next_avail_desc = ctxd;
3479 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3480 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3481 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3488 ** This routine parses packet headers so that Flow
3489 ** Director can make a hashed filter table entry
3490 ** allowing traffic flows to be identified and kept
3491 ** on the same cpu. This would be a performance
3492 ** hit, but we only do it at IXGBE_FDIR_RATE of
3496 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3498 struct adapter *adapter = txr->adapter;
3499 struct ix_queue *que;
3503 struct ether_vlan_header *eh;
3504 union ixgbe_atr_hash_dword input = {.dword = 0};
3505 union ixgbe_atr_hash_dword common = {.dword = 0};
3506 int ehdrlen, ip_hlen;
3509 eh = mtod(mp, struct ether_vlan_header *);
3510 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3511 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3512 etype = eh->evl_proto;
3514 ehdrlen = ETHER_HDR_LEN;
3515 etype = eh->evl_encap_proto;
3518 /* Only handling IPv4 */
3519 if (etype != htons(ETHERTYPE_IP))
3522 ip = (struct ip *)(mp->m_data + ehdrlen);
3523 ip_hlen = ip->ip_hl << 2;
3525 /* check if we're UDP or TCP */
3528 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3529 /* src and dst are inverted */
3530 common.port.dst ^= th->th_sport;
3531 common.port.src ^= th->th_dport;
3532 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3535 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3536 /* src and dst are inverted */
3537 common.port.dst ^= uh->uh_sport;
3538 common.port.src ^= uh->uh_dport;
3539 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3545 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3546 if (mp->m_pkthdr.ether_vtag)
3547 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3549 common.flex_bytes ^= etype;
3550 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3552 que = &adapter->queues[txr->me];
3554 ** This assumes the Rx queue and Tx
3555 ** queue are bound to the same CPU
3557 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3558 input, common, que->msix);
3560 #endif /* IXGBE_FDIR */
3562 /**********************************************************************
3564 * Examine each tx_buffer in the used queue. If the hardware is done
3565 * processing the packet then free associated resources. The
3566 * tx_buffer is put back on the free queue.
3568 **********************************************************************/
3570 ixgbe_txeof(struct tx_ring *txr)
3572 struct adapter *adapter = txr->adapter;
3573 struct ifnet *ifp = adapter->ifp;
3574 u32 work, processed = 0;
3575 u16 limit = txr->process_limit;
3576 struct ixgbe_tx_buf *buf;
3577 union ixgbe_adv_tx_desc *txd;
3579 mtx_assert(&txr->tx_mtx, MA_OWNED);
3582 if (ifp->if_capenable & IFCAP_NETMAP) {
3583 struct netmap_adapter *na = NA(ifp);
3584 struct netmap_kring *kring = &na->tx_rings[txr->me];
3586 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3587 BUS_DMASYNC_POSTREAD);
3589 * In netmap mode, all the work is done in the context
3590 * of the client thread. Interrupt handlers only wake up
3591 * clients, which may be sleeping on individual rings
3592 * or on a global resource for all rings.
3593 * To implement tx interrupt mitigation, we wake up the client
3594 * thread roughly every half ring, even if the NIC interrupts
3595 * more frequently. This is implemented as follows:
3596 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3597 * the slot that should wake up the thread (nkr_num_slots
3598 * means the user thread should not be woken up);
3599 * - the driver ignores tx interrupts unless netmap_mitigate=0
3600 * or the slot has the DD bit set.
3602 * When the driver has separate locks, we need to
3603 * release and re-acquire txlock to avoid deadlocks.
3604 * XXX see if we can find a better way.
3606 if (!netmap_mitigate ||
3607 (kring->nr_kflags < kring->nkr_num_slots &&
3608 txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3609 netmap_tx_irq(ifp, txr->me |
3610 (NETMAP_LOCKED_ENTER|NETMAP_LOCKED_EXIT));
3614 #endif /* DEV_NETMAP */
3616 if (txr->tx_avail == txr->num_desc) {
3617 txr->queue_status = IXGBE_QUEUE_IDLE;
3621 /* Get work starting point */
3622 work = txr->next_to_clean;
3623 buf = &txr->tx_buffers[work];
3624 txd = &txr->tx_base[work];
3625 work -= txr->num_desc; /* The distance to ring end */
3626 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3627 BUS_DMASYNC_POSTREAD);
3630 union ixgbe_adv_tx_desc *eop= buf->eop;
3631 if (eop == NULL) /* No work */
3634 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3635 break; /* I/O not complete */
3639 buf->m_head->m_pkthdr.len;
3640 bus_dmamap_sync(txr->txtag,
3642 BUS_DMASYNC_POSTWRITE);
3643 bus_dmamap_unload(txr->txtag,
3645 m_freem(buf->m_head);
3652 /* We clean the range if multi segment */
3653 while (txd != eop) {
3657 /* wrap the ring? */
3658 if (__predict_false(!work)) {
3659 work -= txr->num_desc;
3660 buf = txr->tx_buffers;
3665 buf->m_head->m_pkthdr.len;
3666 bus_dmamap_sync(txr->txtag,
3668 BUS_DMASYNC_POSTWRITE);
3669 bus_dmamap_unload(txr->txtag,
3671 m_freem(buf->m_head);
3682 txr->watchdog_time = ticks;
3684 /* Try the next packet */
3688 /* reset with a wrap */
3689 if (__predict_false(!work)) {
3690 work -= txr->num_desc;
3691 buf = txr->tx_buffers;
3695 } while (__predict_true(--limit));
3697 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3698 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3700 work += txr->num_desc;
3701 txr->next_to_clean = work;
3704 ** Watchdog calculation, we know there's
3705 ** work outstanding or the first return
3706 ** would have been taken, so none processed
3707 ** for too long indicates a hang.
3709 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3710 txr->queue_status = IXGBE_QUEUE_HUNG;
3712 if (txr->tx_avail == txr->num_desc)
3713 txr->queue_status = IXGBE_QUEUE_IDLE;
3718 /*********************************************************************
3720 * Refresh mbuf buffers for RX descriptor rings
3721 * - now keeps its own state so discards due to resource
3722 * exhaustion are unnecessary, if an mbuf cannot be obtained
3723 * it just returns, keeping its placeholder, thus it can simply
3724 * be recalled to try again.
3726 **********************************************************************/
3728 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3730 struct adapter *adapter = rxr->adapter;
3731 bus_dma_segment_t seg[1];
3732 struct ixgbe_rx_buf *rxbuf;
3734 int i, j, nsegs, error;
3735 bool refreshed = FALSE;
3737 i = j = rxr->next_to_refresh;
3738 /* Control the loop with one beyond */
3739 if (++j == rxr->num_desc)
3742 while (j != limit) {
3743 rxbuf = &rxr->rx_buffers[i];
3744 if (rxbuf->buf == NULL) {
3745 mp = m_getjcl(M_NOWAIT, MT_DATA,
3746 M_PKTHDR, rxr->mbuf_sz);
3749 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3750 m_adj(mp, ETHER_ALIGN);
3754 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3756 /* If we're dealing with an mbuf that was copied rather
3757 * than replaced, there's no need to go through busdma.
3759 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3760 /* Get the memory mapping */
3761 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3762 rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3764 printf("Refresh mbufs: payload dmamap load"
3765 " failure - %d\n", error);
3771 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3772 BUS_DMASYNC_PREREAD);
3773 rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3774 htole64(seg[0].ds_addr);
3776 rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3777 rxbuf->flags &= ~IXGBE_RX_COPY;
3781 /* Next is precalculated */
3783 rxr->next_to_refresh = i;
3784 if (++j == rxr->num_desc)
3788 if (refreshed) /* Update hardware tail index */
3789 IXGBE_WRITE_REG(&adapter->hw,
3790 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3794 /*********************************************************************
3796 * Allocate memory for rx_buffer structures. Since we use one
3797 * rx_buffer per received packet, the maximum number of rx_buffer's
3798 * that we'll need is equal to the number of receive descriptors
3799 * that we've allocated.
3801 **********************************************************************/
3803 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3805 struct adapter *adapter = rxr->adapter;
3806 device_t dev = adapter->dev;
3807 struct ixgbe_rx_buf *rxbuf;
3808 int i, bsize, error;
3810 bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3811 if (!(rxr->rx_buffers =
3812 (struct ixgbe_rx_buf *) malloc(bsize,
3813 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3814 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3819 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3820 1, 0, /* alignment, bounds */
3821 BUS_SPACE_MAXADDR, /* lowaddr */
3822 BUS_SPACE_MAXADDR, /* highaddr */
3823 NULL, NULL, /* filter, filterarg */
3824 MJUM16BYTES, /* maxsize */
3826 MJUM16BYTES, /* maxsegsize */
3828 NULL, /* lockfunc */
3829 NULL, /* lockfuncarg */
3831 device_printf(dev, "Unable to create RX DMA tag\n");
3835 for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3836 rxbuf = &rxr->rx_buffers[i];
3837 error = bus_dmamap_create(rxr->ptag,
3838 BUS_DMA_NOWAIT, &rxbuf->pmap);
3840 device_printf(dev, "Unable to create RX dma map\n");
3848 /* Frees all, but can handle partial completion */
3849 ixgbe_free_receive_structures(adapter);
3854 ** Used to detect a descriptor that has
3855 ** been merged by Hardware RSC.
3858 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3860 return (le32toh(rx->wb.lower.lo_dword.data) &
3861 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3864 /*********************************************************************
3866 * Initialize Hardware RSC (LRO) feature on 82599
3867 * for an RX ring, this is toggled by the LRO capability
3868 * even though it is transparent to the stack.
3870 * NOTE: since this HW feature only works with IPV4 and
3871 * our testing has shown soft LRO to be as effective
3872 * I have decided to disable this by default.
3874 **********************************************************************/
3876 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3878 struct adapter *adapter = rxr->adapter;
3879 struct ixgbe_hw *hw = &adapter->hw;
3880 u32 rscctrl, rdrxctl;
3882 /* If turning LRO/RSC off we need to disable it */
3883 if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3884 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3885 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3889 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3890 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3891 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3892 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3893 #endif /* DEV_NETMAP */
3894 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3895 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3896 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3898 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3899 rscctrl |= IXGBE_RSCCTL_RSCEN;
3901 ** Limit the total number of descriptors that
3902 ** can be combined, so it does not exceed 64K
3904 if (rxr->mbuf_sz == MCLBYTES)
3905 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3906 else if (rxr->mbuf_sz == MJUMPAGESIZE)
3907 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3908 else if (rxr->mbuf_sz == MJUM9BYTES)
3909 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3910 else /* Using 16K cluster */
3911 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3913 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3915 /* Enable TCP header recognition */
3916 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3917 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3918 IXGBE_PSRTYPE_TCPHDR));
3920 /* Disable RSC for ACK packets */
3921 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3922 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3929 ixgbe_free_receive_ring(struct rx_ring *rxr)
3931 struct ixgbe_rx_buf *rxbuf;
3934 for (i = 0; i < rxr->num_desc; i++) {
3935 rxbuf = &rxr->rx_buffers[i];
3936 if (rxbuf->buf != NULL) {
3937 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3938 BUS_DMASYNC_POSTREAD);
3939 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3940 rxbuf->buf->m_flags |= M_PKTHDR;
3941 m_freem(rxbuf->buf);
3948 /*********************************************************************
3950 * Initialize a receive ring and its buffers.
3952 **********************************************************************/
3954 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3956 struct adapter *adapter;
3959 struct ixgbe_rx_buf *rxbuf;
3960 bus_dma_segment_t seg[1];
3961 struct lro_ctrl *lro = &rxr->lro;
3962 int rsize, nsegs, error = 0;
3964 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3965 struct netmap_slot *slot;
3966 #endif /* DEV_NETMAP */
3968 adapter = rxr->adapter;
3972 /* Clear the ring contents */
3975 /* same as in ixgbe_setup_transmit_ring() */
3976 slot = netmap_reset(na, NR_RX, rxr->me, 0);
3977 #endif /* DEV_NETMAP */
3978 rsize = roundup2(adapter->num_rx_desc *
3979 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3980 bzero((void *)rxr->rx_base, rsize);
3981 /* Cache the size */
3982 rxr->mbuf_sz = adapter->rx_mbuf_sz;
3984 /* Free current RX buffer structs and their mbufs */
3985 ixgbe_free_receive_ring(rxr);
3987 /* Now replenish the mbufs */
3988 for (int j = 0; j != rxr->num_desc; ++j) {
3991 rxbuf = &rxr->rx_buffers[j];
3994 * In netmap mode, fill the map and set the buffer
3995 * address in the NIC ring, considering the offset
3996 * between the netmap and NIC rings (see comment in
3997 * ixgbe_setup_transmit_ring() ). No need to allocate
3998 * an mbuf, so end the block with a continue;
4001 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4005 addr = PNMB(slot + sj, &paddr);
4006 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4007 /* Update descriptor */
4008 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4011 #endif /* DEV_NETMAP */
4012 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
4013 M_PKTHDR, adapter->rx_mbuf_sz);
4014 if (rxbuf->buf == NULL) {
4019 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4020 /* Get the memory mapping */
4021 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4022 rxbuf->pmap, mp, seg,
4023 &nsegs, BUS_DMA_NOWAIT);
4026 bus_dmamap_sync(rxr->ptag,
4027 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4028 /* Update descriptor */
4029 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4033 /* Setup our descriptor indices */
4034 rxr->next_to_check = 0;
4035 rxr->next_to_refresh = 0;
4036 rxr->lro_enabled = FALSE;
4039 rxr->discard = FALSE;
4040 rxr->vtag_strip = FALSE;
4042 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4043 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4046 ** Now set up the LRO interface:
4048 if (ixgbe_rsc_enable)
4049 ixgbe_setup_hw_rsc(rxr);
4050 else if (ifp->if_capenable & IFCAP_LRO) {
4051 int err = tcp_lro_init(lro);
4053 device_printf(dev, "LRO Initialization failed!\n");
4056 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4057 rxr->lro_enabled = TRUE;
4058 lro->ifp = adapter->ifp;
4061 IXGBE_RX_UNLOCK(rxr);
4065 ixgbe_free_receive_ring(rxr);
4066 IXGBE_RX_UNLOCK(rxr);
4070 /*********************************************************************
4072 * Initialize all receive rings.
4074 **********************************************************************/
4076 ixgbe_setup_receive_structures(struct adapter *adapter)
4078 struct rx_ring *rxr = adapter->rx_rings;
4081 for (j = 0; j < adapter->num_queues; j++, rxr++)
4082 if (ixgbe_setup_receive_ring(rxr))
4088 * Free RX buffers allocated so far, we will only handle
4089 * the rings that completed, the failing case will have
4090 * cleaned up for itself. 'j' failed, so its the terminus.
4092 for (int i = 0; i < j; ++i) {
4093 rxr = &adapter->rx_rings[i];
4094 ixgbe_free_receive_ring(rxr);
4100 /*********************************************************************
4102 * Setup receive registers and features.
4104 **********************************************************************/
4105 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4107 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4110 ixgbe_initialize_receive_units(struct adapter *adapter)
4112 struct rx_ring *rxr = adapter->rx_rings;
4113 struct ixgbe_hw *hw = &adapter->hw;
4114 struct ifnet *ifp = adapter->ifp;
4115 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
4116 u32 reta, mrqc = 0, hlreg, random[10];
4120 * Make sure receives are disabled while
4121 * setting up the descriptor ring
4123 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4124 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4125 rxctrl & ~IXGBE_RXCTRL_RXEN);
4127 /* Enable broadcasts */
4128 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4129 fctrl |= IXGBE_FCTRL_BAM;
4130 fctrl |= IXGBE_FCTRL_DPF;
4131 fctrl |= IXGBE_FCTRL_PMCF;
4132 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4134 /* Set for Jumbo Frames? */
4135 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4136 if (ifp->if_mtu > ETHERMTU)
4137 hlreg |= IXGBE_HLREG0_JUMBOEN;
4139 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4141 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4142 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4143 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4145 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4146 #endif /* DEV_NETMAP */
4147 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4149 bufsz = (adapter->rx_mbuf_sz +
4150 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4152 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4153 u64 rdba = rxr->rxdma.dma_paddr;
4155 /* Setup the Base and Length of the Rx Descriptor Ring */
4156 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4157 (rdba & 0x00000000ffffffffULL));
4158 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4159 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4160 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4162 /* Set up the SRRCTL register */
4163 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4164 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4165 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4167 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4168 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4170 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4171 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4172 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4174 /* Set the processing limit */
4175 rxr->process_limit = ixgbe_rx_process_limit;
4178 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4179 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4180 IXGBE_PSRTYPE_UDPHDR |
4181 IXGBE_PSRTYPE_IPV4HDR |
4182 IXGBE_PSRTYPE_IPV6HDR;
4183 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4186 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4189 if (adapter->num_queues > 1) {
4193 /* set up random bits */
4194 arc4rand(&random, sizeof(random), 0);
4196 /* Set up the redirection table */
4197 for (i = 0, j = 0; i < 128; i++, j++) {
4198 if (j == adapter->num_queues) j = 0;
4199 reta = (reta << 8) | (j * 0x11);
4201 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4204 /* Now fill our hash function seeds */
4205 for (int i = 0; i < 10; i++)
4206 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4208 /* Perform hash on these packet types */
4209 mrqc = IXGBE_MRQC_RSSEN
4210 | IXGBE_MRQC_RSS_FIELD_IPV4
4211 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4212 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4213 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4214 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4215 | IXGBE_MRQC_RSS_FIELD_IPV6
4216 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4217 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4218 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4219 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4221 /* RSS and RX IPP Checksum are mutually exclusive */
4222 rxcsum |= IXGBE_RXCSUM_PCSD;
4225 if (ifp->if_capenable & IFCAP_RXCSUM)
4226 rxcsum |= IXGBE_RXCSUM_PCSD;
4228 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4229 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4231 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4236 /*********************************************************************
4238 * Free all receive rings.
4240 **********************************************************************/
4242 ixgbe_free_receive_structures(struct adapter *adapter)
4244 struct rx_ring *rxr = adapter->rx_rings;
4246 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4247 struct lro_ctrl *lro = &rxr->lro;
4248 ixgbe_free_receive_buffers(rxr);
4249 /* Free LRO memory */
4251 /* Free the ring memory as well */
4252 ixgbe_dma_free(adapter, &rxr->rxdma);
4255 free(adapter->rx_rings, M_DEVBUF);
4259 /*********************************************************************
4261 * Free receive ring data structures
4263 **********************************************************************/
4265 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4267 struct adapter *adapter = rxr->adapter;
4268 struct ixgbe_rx_buf *rxbuf;
4270 INIT_DEBUGOUT("free_receive_structures: begin");
4272 /* Cleanup any existing buffers */
4273 if (rxr->rx_buffers != NULL) {
4274 for (int i = 0; i < adapter->num_rx_desc; i++) {
4275 rxbuf = &rxr->rx_buffers[i];
4276 if (rxbuf->buf != NULL) {
4277 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4278 BUS_DMASYNC_POSTREAD);
4279 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4280 rxbuf->buf->m_flags |= M_PKTHDR;
4281 m_freem(rxbuf->buf);
4284 if (rxbuf->pmap != NULL) {
4285 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4289 if (rxr->rx_buffers != NULL) {
4290 free(rxr->rx_buffers, M_DEVBUF);
4291 rxr->rx_buffers = NULL;
4295 if (rxr->ptag != NULL) {
4296 bus_dma_tag_destroy(rxr->ptag);
4303 static __inline void
4304 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4308 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4309 * should be computed by hardware. Also it should not have VLAN tag in
4310 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4312 if (rxr->lro_enabled &&
4313 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4314 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4315 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4316 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4317 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4318 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4319 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4320 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4322 * Send to the stack if:
4323 ** - LRO not enabled, or
4324 ** - no LRO resources, or
4325 ** - lro enqueue fails
4327 if (rxr->lro.lro_cnt != 0)
4328 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4331 IXGBE_RX_UNLOCK(rxr);
4332 (*ifp->if_input)(ifp, m);
4336 static __inline void
4337 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4339 struct ixgbe_rx_buf *rbuf;
4341 rbuf = &rxr->rx_buffers[i];
4343 if (rbuf->fmp != NULL) {/* Partial chain ? */
4344 rbuf->fmp->m_flags |= M_PKTHDR;
4350 ** With advanced descriptors the writeback
4351 ** clobbers the buffer addrs, so its easier
4352 ** to just free the existing mbufs and take
4353 ** the normal refresh path to get new buffers
4365 /*********************************************************************
4367 * This routine executes in interrupt context. It replenishes
4368 * the mbufs in the descriptor and sends data which has been
4369 * dma'ed into host memory to upper layer.
4371 * We loop at most count times if count is > 0, or until done if
4374 * Return TRUE for more work, FALSE for all clean.
4375 *********************************************************************/
4377 ixgbe_rxeof(struct ix_queue *que)
4379 struct adapter *adapter = que->adapter;
4380 struct rx_ring *rxr = que->rxr;
4381 struct ifnet *ifp = adapter->ifp;
4382 struct lro_ctrl *lro = &rxr->lro;
4383 struct lro_entry *queued;
4384 int i, nextp, processed = 0;
4386 u16 count = rxr->process_limit;
4387 union ixgbe_adv_rx_desc *cur;
4388 struct ixgbe_rx_buf *rbuf, *nbuf;
4393 /* Same as the txeof routine: wakeup clients on intr. */
4394 if (netmap_rx_irq(ifp, rxr->me | NETMAP_LOCKED_ENTER, &processed))
4396 #endif /* DEV_NETMAP */
4398 for (i = rxr->next_to_check; count != 0;) {
4399 struct mbuf *sendmp, *mp;
4405 /* Sync the ring. */
4406 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4407 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4409 cur = &rxr->rx_base[i];
4410 staterr = le32toh(cur->wb.upper.status_error);
4412 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4414 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4421 cur->wb.upper.status_error = 0;
4422 rbuf = &rxr->rx_buffers[i];
4425 len = le16toh(cur->wb.upper.length);
4426 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4427 IXGBE_RXDADV_PKTTYPE_MASK;
4428 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4430 /* Make sure bad packets are discarded */
4431 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4433 rxr->rx_discarded++;
4435 rxr->discard = FALSE;
4437 rxr->discard = TRUE;
4438 ixgbe_rx_discard(rxr, i);
4443 ** On 82599 which supports a hardware
4444 ** LRO (called HW RSC), packets need
4445 ** not be fragmented across sequential
4446 ** descriptors, rather the next descriptor
4447 ** is indicated in bits of the descriptor.
4448 ** This also means that we might proceses
4449 ** more than one packet at a time, something
4450 ** that has never been true before, it
4451 ** required eliminating global chain pointers
4452 ** in favor of what we are doing here. -jfv
4456 ** Figure out the next descriptor
4459 if (rxr->hw_rsc == TRUE) {
4460 rsc = ixgbe_rsc_count(cur);
4461 rxr->rsc_num += (rsc - 1);
4463 if (rsc) { /* Get hardware index */
4465 IXGBE_RXDADV_NEXTP_MASK) >>
4466 IXGBE_RXDADV_NEXTP_SHIFT);
4467 } else { /* Just sequential */
4469 if (nextp == adapter->num_rx_desc)
4472 nbuf = &rxr->rx_buffers[nextp];
4476 ** Rather than using the fmp/lmp global pointers
4477 ** we now keep the head of a packet chain in the
4478 ** buffer struct and pass this along from one
4479 ** descriptor to the next, until we get EOP.
4483 ** See if there is a stored head
4484 ** that determines what we are
4487 if (sendmp != NULL) { /* secondary frag */
4488 rbuf->buf = rbuf->fmp = NULL;
4489 mp->m_flags &= ~M_PKTHDR;
4490 sendmp->m_pkthdr.len += mp->m_len;
4493 * Optimize. This might be a small packet,
4494 * maybe just a TCP ACK. Do a fast copy that
4495 * is cache aligned into a new mbuf, and
4496 * leave the old mbuf+cluster for re-use.
4498 if (eop && len <= IXGBE_RX_COPY_LEN) {
4499 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4500 if (sendmp != NULL) {
4502 IXGBE_RX_COPY_ALIGN;
4503 ixgbe_bcopy(mp->m_data,
4504 sendmp->m_data, len);
4505 sendmp->m_len = len;
4507 rbuf->flags |= IXGBE_RX_COPY;
4510 if (sendmp == NULL) {
4511 rbuf->buf = rbuf->fmp = NULL;
4515 /* first desc of a non-ps chain */
4516 sendmp->m_flags |= M_PKTHDR;
4517 sendmp->m_pkthdr.len = mp->m_len;
4521 /* Pass the head pointer on */
4525 mp->m_next = nbuf->buf;
4526 } else { /* Sending this frame */
4527 sendmp->m_pkthdr.rcvif = ifp;
4530 /* capture data for AIM */
4531 rxr->bytes += sendmp->m_pkthdr.len;
4532 rxr->rx_bytes += sendmp->m_pkthdr.len;
4533 /* Process vlan info */
4534 if ((rxr->vtag_strip) &&
4535 (staterr & IXGBE_RXD_STAT_VP))
4536 vtag = le16toh(cur->wb.upper.vlan);
4538 sendmp->m_pkthdr.ether_vtag = vtag;
4539 sendmp->m_flags |= M_VLANTAG;
4541 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4542 ixgbe_rx_checksum(staterr, sendmp, ptype);
4543 #if __FreeBSD_version >= 800000
4544 sendmp->m_pkthdr.flowid = que->msix;
4545 sendmp->m_flags |= M_FLOWID;
4549 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4550 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4552 /* Advance our pointers to the next descriptor. */
4553 if (++i == rxr->num_desc)
4556 /* Now send to the stack or do LRO */
4557 if (sendmp != NULL) {
4558 rxr->next_to_check = i;
4559 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4560 i = rxr->next_to_check;
4563 /* Every 8 descriptors we go to refresh mbufs */
4564 if (processed == 8) {
4565 ixgbe_refresh_mbufs(rxr, i);
4570 /* Refresh any remaining buf structs */
4571 if (ixgbe_rx_unrefreshed(rxr))
4572 ixgbe_refresh_mbufs(rxr, i);
4574 rxr->next_to_check = i;
4577 * Flush any outstanding LRO work
4579 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4580 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4581 tcp_lro_flush(lro, queued);
4584 IXGBE_RX_UNLOCK(rxr);
4587 ** Still have cleaning to do?
4589 if ((staterr & IXGBE_RXD_STAT_DD) != 0)
4596 /*********************************************************************
4598 * Verify that the hardware indicated that the checksum is valid.
4599 * Inform the stack about the status of checksum so that stack
4600 * doesn't spend time verifying the checksum.
4602 *********************************************************************/
4604 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4606 u16 status = (u16) staterr;
4607 u8 errors = (u8) (staterr >> 24);
4610 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4611 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4614 if (status & IXGBE_RXD_STAT_IPCS) {
4615 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4616 /* IP Checksum Good */
4617 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4618 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4621 mp->m_pkthdr.csum_flags = 0;
4623 if (status & IXGBE_RXD_STAT_L4CS) {
4624 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4625 #if __FreeBSD_version >= 800000
4627 type = CSUM_SCTP_VALID;
4629 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4630 mp->m_pkthdr.csum_flags |= type;
4632 mp->m_pkthdr.csum_data = htons(0xffff);
4640 ** This routine is run via an vlan config EVENT,
4641 ** it enables us to use the HW Filter table since
4642 ** we can get the vlan id. This just creates the
4643 ** entry in the soft version of the VFTA, init will
4644 ** repopulate the real table.
4647 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4649 struct adapter *adapter = ifp->if_softc;
4652 if (ifp->if_softc != arg) /* Not our event */
4655 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4658 IXGBE_CORE_LOCK(adapter);
4659 index = (vtag >> 5) & 0x7F;
4661 adapter->shadow_vfta[index] |= (1 << bit);
4662 ++adapter->num_vlans;
4663 ixgbe_init_locked(adapter);
4664 IXGBE_CORE_UNLOCK(adapter);
4668 ** This routine is run via an vlan
4669 ** unconfig EVENT, remove our entry
4670 ** in the soft vfta.
4673 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4675 struct adapter *adapter = ifp->if_softc;
4678 if (ifp->if_softc != arg)
4681 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4684 IXGBE_CORE_LOCK(adapter);
4685 index = (vtag >> 5) & 0x7F;
4687 adapter->shadow_vfta[index] &= ~(1 << bit);
4688 --adapter->num_vlans;
4689 /* Re-init to load the changes */
4690 ixgbe_init_locked(adapter);
4691 IXGBE_CORE_UNLOCK(adapter);
4695 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4697 struct ifnet *ifp = adapter->ifp;
4698 struct ixgbe_hw *hw = &adapter->hw;
4699 struct rx_ring *rxr;
4704 ** We get here thru init_locked, meaning
4705 ** a soft reset, this has already cleared
4706 ** the VFTA and other state, so if there
4707 ** have been no vlan's registered do nothing.
4709 if (adapter->num_vlans == 0)
4713 ** A soft reset zero's out the VFTA, so
4714 ** we need to repopulate it now.
4716 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4717 if (adapter->shadow_vfta[i] != 0)
4718 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4719 adapter->shadow_vfta[i]);
4721 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4722 /* Enable the Filter Table if enabled */
4723 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4724 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4725 ctrl |= IXGBE_VLNCTRL_VFE;
4727 if (hw->mac.type == ixgbe_mac_82598EB)
4728 ctrl |= IXGBE_VLNCTRL_VME;
4729 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4731 /* Setup the queues for vlans */
4732 for (int i = 0; i < adapter->num_queues; i++) {
4733 rxr = &adapter->rx_rings[i];
4734 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4735 if (hw->mac.type != ixgbe_mac_82598EB) {
4736 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4737 ctrl |= IXGBE_RXDCTL_VME;
4738 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4740 rxr->vtag_strip = TRUE;
4745 ixgbe_enable_intr(struct adapter *adapter)
4747 struct ixgbe_hw *hw = &adapter->hw;
4748 struct ix_queue *que = adapter->queues;
4751 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4752 /* Enable Fan Failure detection */
4753 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4754 mask |= IXGBE_EIMS_GPI_SDP1;
4756 switch (adapter->hw.mac.type) {
4757 case ixgbe_mac_82599EB:
4758 mask |= IXGBE_EIMS_ECC;
4759 mask |= IXGBE_EIMS_GPI_SDP0;
4760 mask |= IXGBE_EIMS_GPI_SDP1;
4761 mask |= IXGBE_EIMS_GPI_SDP2;
4763 mask |= IXGBE_EIMS_FLOW_DIR;
4766 case ixgbe_mac_X540:
4767 mask |= IXGBE_EIMS_ECC;
4768 /* Detect if Thermal Sensor is enabled */
4769 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
4770 if (fwsm & IXGBE_FWSM_TS_ENABLED)
4771 mask |= IXGBE_EIMS_TS;
4773 mask |= IXGBE_EIMS_FLOW_DIR;
4780 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4782 /* With RSS we use auto clear */
4783 if (adapter->msix_mem) {
4784 mask = IXGBE_EIMS_ENABLE_MASK;
4785 /* Don't autoclear Link */
4786 mask &= ~IXGBE_EIMS_OTHER;
4787 mask &= ~IXGBE_EIMS_LSC;
4788 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4792 ** Now enable all queues, this is done separately to
4793 ** allow for handling the extended (beyond 32) MSIX
4794 ** vectors that can be used by 82599
4796 for (int i = 0; i < adapter->num_queues; i++, que++)
4797 ixgbe_enable_queue(adapter, que->msix);
4799 IXGBE_WRITE_FLUSH(hw);
4805 ixgbe_disable_intr(struct adapter *adapter)
4807 if (adapter->msix_mem)
4808 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4809 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4810 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4812 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4813 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4814 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4816 IXGBE_WRITE_FLUSH(&adapter->hw);
4821 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4825 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4832 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4834 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4841 ** Get the width and transaction speed of
4842 ** the slot this adapter is plugged into.
4845 ixgbe_get_slot_info(struct ixgbe_hw *hw)
4847 device_t dev = ((struct ixgbe_osdep *)hw->back)->dev;
4848 struct ixgbe_mac_info *mac = &hw->mac;
4852 /* For most devices simply call the shared code routine */
4853 if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
4854 ixgbe_get_bus_info(hw);
4859 ** For the Quad port adapter we need to parse back
4860 ** up the PCI tree to find the speed of the expansion
4861 ** slot into which this adapter is plugged. A bit more work.
4863 dev = device_get_parent(device_get_parent(dev));
4865 device_printf(dev, "parent pcib = %x,%x,%x\n",
4866 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4868 dev = device_get_parent(device_get_parent(dev));
4870 device_printf(dev, "slot pcib = %x,%x,%x\n",
4871 pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4873 /* Now get the PCI Express Capabilities offset */
4874 pci_find_cap(dev, PCIY_EXPRESS, &offset);
4875 /* ...and read the Link Status Register */
4876 link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4877 switch (link & IXGBE_PCI_LINK_WIDTH) {
4878 case IXGBE_PCI_LINK_WIDTH_1:
4879 hw->bus.width = ixgbe_bus_width_pcie_x1;
4881 case IXGBE_PCI_LINK_WIDTH_2:
4882 hw->bus.width = ixgbe_bus_width_pcie_x2;
4884 case IXGBE_PCI_LINK_WIDTH_4:
4885 hw->bus.width = ixgbe_bus_width_pcie_x4;
4887 case IXGBE_PCI_LINK_WIDTH_8:
4888 hw->bus.width = ixgbe_bus_width_pcie_x8;
4891 hw->bus.width = ixgbe_bus_width_unknown;
4895 switch (link & IXGBE_PCI_LINK_SPEED) {
4896 case IXGBE_PCI_LINK_SPEED_2500:
4897 hw->bus.speed = ixgbe_bus_speed_2500;
4899 case IXGBE_PCI_LINK_SPEED_5000:
4900 hw->bus.speed = ixgbe_bus_speed_5000;
4902 case IXGBE_PCI_LINK_SPEED_8000:
4903 hw->bus.speed = ixgbe_bus_speed_8000;
4906 hw->bus.speed = ixgbe_bus_speed_unknown;
4910 mac->ops.set_lan_id(hw);
4913 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4914 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
4915 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
4916 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4917 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
4918 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
4919 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
4922 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4923 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
4924 (hw->bus.speed == ixgbe_bus_speed_2500))) {
4925 device_printf(dev, "PCI-Express bandwidth available"
4926 " for this card\n is not sufficient for"
4927 " optimal performance.\n");
4928 device_printf(dev, "For optimal performance a x8 "
4929 "PCIE, or x4 PCIE Gen2 slot is required.\n");
4931 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4932 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
4933 (hw->bus.speed < ixgbe_bus_speed_8000))) {
4934 device_printf(dev, "PCI-Express bandwidth available"
4935 " for this card\n is not sufficient for"
4936 " optimal performance.\n");
4937 device_printf(dev, "For optimal performance a x8 "
4938 "PCIE Gen3 slot is required.\n");
4946 ** Setup the correct IVAR register for a particular MSIX interrupt
4947 ** (yes this is all very magic and confusing :)
4948 ** - entry is the register array entry
4949 ** - vector is the MSIX vector for this queue
4950 ** - type is RX/TX/MISC
4953 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4955 struct ixgbe_hw *hw = &adapter->hw;
4958 vector |= IXGBE_IVAR_ALLOC_VAL;
4960 switch (hw->mac.type) {
4962 case ixgbe_mac_82598EB:
4964 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4966 entry += (type * 64);
4967 index = (entry >> 2) & 0x1F;
4968 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4969 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4970 ivar |= (vector << (8 * (entry & 0x3)));
4971 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4974 case ixgbe_mac_82599EB:
4975 case ixgbe_mac_X540:
4976 if (type == -1) { /* MISC IVAR */
4977 index = (entry & 1) * 8;
4978 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4979 ivar &= ~(0xFF << index);
4980 ivar |= (vector << index);
4981 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4982 } else { /* RX/TX IVARS */
4983 index = (16 * (entry & 1)) + (8 * type);
4984 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4985 ivar &= ~(0xFF << index);
4986 ivar |= (vector << index);
4987 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4996 ixgbe_configure_ivars(struct adapter *adapter)
4998 struct ix_queue *que = adapter->queues;
5001 if (ixgbe_max_interrupt_rate > 0)
5002 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5006 for (int i = 0; i < adapter->num_queues; i++, que++) {
5007 /* First the RX queue entry */
5008 ixgbe_set_ivar(adapter, i, que->msix, 0);
5009 /* ... and the TX */
5010 ixgbe_set_ivar(adapter, i, que->msix, 1);
5011 /* Set an Initial EITR value */
5012 IXGBE_WRITE_REG(&adapter->hw,
5013 IXGBE_EITR(que->msix), newitr);
5016 /* For the Link interrupt */
5017 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5021 ** ixgbe_sfp_probe - called in the local timer to
5022 ** determine if a port had optics inserted.
5024 static bool ixgbe_sfp_probe(struct adapter *adapter)
5026 struct ixgbe_hw *hw = &adapter->hw;
5027 device_t dev = adapter->dev;
5028 bool result = FALSE;
5030 if ((hw->phy.type == ixgbe_phy_nl) &&
5031 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5032 s32 ret = hw->phy.ops.identify_sfp(hw);
5035 ret = hw->phy.ops.reset(hw);
5036 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5037 device_printf(dev,"Unsupported SFP+ module detected!");
5038 printf(" Reload driver with supported module.\n");
5039 adapter->sfp_probe = FALSE;
5042 device_printf(dev,"SFP+ module detected!\n");
5043 /* We now have supported optics */
5044 adapter->sfp_probe = FALSE;
5045 /* Set the optics type so system reports correctly */
5046 ixgbe_setup_optics(adapter);
5054 ** Tasklet handler for MSIX Link interrupts
5055 ** - do outside interrupt since it might sleep
5058 ixgbe_handle_link(void *context, int pending)
5060 struct adapter *adapter = context;
5062 ixgbe_check_link(&adapter->hw,
5063 &adapter->link_speed, &adapter->link_up, 0);
5064 ixgbe_update_link_status(adapter);
5068 ** Tasklet for handling SFP module interrupts
5071 ixgbe_handle_mod(void *context, int pending)
5073 struct adapter *adapter = context;
5074 struct ixgbe_hw *hw = &adapter->hw;
5075 device_t dev = adapter->dev;
5078 err = hw->phy.ops.identify_sfp(hw);
5079 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5081 "Unsupported SFP+ module type was detected.\n");
5084 err = hw->mac.ops.setup_sfp(hw);
5085 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5087 "Setup failure - unsupported SFP+ module type.\n");
5090 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5096 ** Tasklet for handling MSF (multispeed fiber) interrupts
5099 ixgbe_handle_msf(void *context, int pending)
5101 struct adapter *adapter = context;
5102 struct ixgbe_hw *hw = &adapter->hw;
5106 autoneg = hw->phy.autoneg_advertised;
5107 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5108 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5109 if (hw->mac.ops.setup_link)
5110 hw->mac.ops.setup_link(hw, autoneg, TRUE);
5116 ** Tasklet for reinitializing the Flow Director filter table
5119 ixgbe_reinit_fdir(void *context, int pending)
5121 struct adapter *adapter = context;
5122 struct ifnet *ifp = adapter->ifp;
5124 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5126 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5127 adapter->fdir_reinit = 0;
5128 /* re-enable flow director interrupts */
5129 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5130 /* Restart the interface */
5131 ifp->if_drv_flags |= IFF_DRV_RUNNING;
5136 /**********************************************************************
5138 * Update the board statistics counters.
5140 **********************************************************************/
5142 ixgbe_update_stats_counters(struct adapter *adapter)
5144 struct ifnet *ifp = adapter->ifp;
5145 struct ixgbe_hw *hw = &adapter->hw;
5146 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5147 u64 total_missed_rx = 0;
5149 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5150 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5151 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5152 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5155 ** Note: these are for the 8 possible traffic classes,
5156 ** which in current implementation is unused,
5157 ** therefore only 0 should read real data.
5159 for (int i = 0; i < 8; i++) {
5161 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5162 /* missed_rx tallies misses for the gprc workaround */
5164 /* global total per queue */
5165 adapter->stats.mpc[i] += mp;
5166 /* Running comprehensive total for stats display */
5167 total_missed_rx += adapter->stats.mpc[i];
5168 if (hw->mac.type == ixgbe_mac_82598EB) {
5169 adapter->stats.rnbc[i] +=
5170 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5171 adapter->stats.qbtc[i] +=
5172 IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5173 adapter->stats.qbrc[i] +=
5174 IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5175 adapter->stats.pxonrxc[i] +=
5176 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5178 adapter->stats.pxonrxc[i] +=
5179 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5180 adapter->stats.pxontxc[i] +=
5181 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5182 adapter->stats.pxofftxc[i] +=
5183 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5184 adapter->stats.pxoffrxc[i] +=
5185 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5186 adapter->stats.pxon2offc[i] +=
5187 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5189 for (int i = 0; i < 16; i++) {
5190 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5191 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5192 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5194 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5195 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5196 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5198 /* Hardware workaround, gprc counts missed packets */
5199 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5200 adapter->stats.gprc -= missed_rx;
5202 if (hw->mac.type != ixgbe_mac_82598EB) {
5203 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5204 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5205 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5206 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5207 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5208 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5209 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5210 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5212 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5213 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5214 /* 82598 only has a counter in the high register */
5215 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5216 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5217 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5221 * Workaround: mprc hardware is incorrectly counting
5222 * broadcasts, so for now we subtract those.
5224 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5225 adapter->stats.bprc += bprc;
5226 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5227 if (hw->mac.type == ixgbe_mac_82598EB)
5228 adapter->stats.mprc -= bprc;
5230 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5231 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5232 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5233 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5234 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5235 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5237 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5238 adapter->stats.lxontxc += lxon;
5239 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5240 adapter->stats.lxofftxc += lxoff;
5241 total = lxon + lxoff;
5243 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5244 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5245 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5246 adapter->stats.gptc -= total;
5247 adapter->stats.mptc -= total;
5248 adapter->stats.ptc64 -= total;
5249 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5251 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5252 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5253 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5254 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5255 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5256 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5257 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5258 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5259 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5260 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5261 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5262 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5263 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5264 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5265 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5266 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5267 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5268 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5269 /* Only read FCOE on 82599 */
5270 if (hw->mac.type != ixgbe_mac_82598EB) {
5271 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5272 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5273 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5274 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5275 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5278 /* Fill out the OS statistics structure */
5279 ifp->if_ipackets = adapter->stats.gprc;
5280 ifp->if_opackets = adapter->stats.gptc;
5281 ifp->if_ibytes = adapter->stats.gorc;
5282 ifp->if_obytes = adapter->stats.gotc;
5283 ifp->if_imcasts = adapter->stats.mprc;
5284 ifp->if_omcasts = adapter->stats.mptc;
5285 ifp->if_collisions = 0;
5288 ifp->if_iqdrops = total_missed_rx;
5289 ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec;
5292 /** ixgbe_sysctl_tdh_handler - Handler function
5293 * Retrieves the TDH value from the hardware
5296 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5300 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5303 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5304 error = sysctl_handle_int(oidp, &val, 0, req);
5305 if (error || !req->newptr)
5310 /** ixgbe_sysctl_tdt_handler - Handler function
5311 * Retrieves the TDT value from the hardware
5314 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5318 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5321 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5322 error = sysctl_handle_int(oidp, &val, 0, req);
5323 if (error || !req->newptr)
5328 /** ixgbe_sysctl_rdh_handler - Handler function
5329 * Retrieves the RDH value from the hardware
5332 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5336 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5339 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5340 error = sysctl_handle_int(oidp, &val, 0, req);
5341 if (error || !req->newptr)
5346 /** ixgbe_sysctl_rdt_handler - Handler function
5347 * Retrieves the RDT value from the hardware
5350 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5354 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5357 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5358 error = sysctl_handle_int(oidp, &val, 0, req);
5359 if (error || !req->newptr)
5365 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5368 struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5369 unsigned int reg, usec, rate;
5371 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5372 usec = ((reg & 0x0FF8) >> 3);
5374 rate = 500000 / usec;
5377 error = sysctl_handle_int(oidp, &rate, 0, req);
5378 if (error || !req->newptr)
5380 reg &= ~0xfff; /* default, no limitation */
5381 ixgbe_max_interrupt_rate = 0;
5382 if (rate > 0 && rate < 500000) {
5385 ixgbe_max_interrupt_rate = rate;
5386 reg |= ((4000000/rate) & 0xff8 );
5388 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5393 * Add sysctl variables, one per statistic, to the system.
5396 ixgbe_add_hw_stats(struct adapter *adapter)
5399 device_t dev = adapter->dev;
5401 struct tx_ring *txr = adapter->tx_rings;
5402 struct rx_ring *rxr = adapter->rx_rings;
5404 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5405 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5406 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5407 struct ixgbe_hw_stats *stats = &adapter->stats;
5409 struct sysctl_oid *stat_node, *queue_node;
5410 struct sysctl_oid_list *stat_list, *queue_list;
5412 #define QUEUE_NAME_LEN 32
5413 char namebuf[QUEUE_NAME_LEN];
5415 /* Driver Statistics */
5416 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5417 CTLFLAG_RD, &adapter->dropped_pkts,
5418 "Driver dropped packets");
5419 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5420 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5421 "m_defrag() failed");
5422 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5423 CTLFLAG_RD, &adapter->watchdog_events,
5424 "Watchdog timeouts");
5425 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5426 CTLFLAG_RD, &adapter->link_irq,
5427 "Link MSIX IRQ Handled");
5429 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5430 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5431 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5432 CTLFLAG_RD, NULL, "Queue Name");
5433 queue_list = SYSCTL_CHILDREN(queue_node);
5435 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5436 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5437 sizeof(&adapter->queues[i]),
5438 ixgbe_sysctl_interrupt_rate_handler, "IU",
5440 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5441 CTLFLAG_RD, &(adapter->queues[i].irqs),
5442 "irqs on this queue");
5443 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5444 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5445 ixgbe_sysctl_tdh_handler, "IU",
5446 "Transmit Descriptor Head");
5447 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5448 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5449 ixgbe_sysctl_tdt_handler, "IU",
5450 "Transmit Descriptor Tail");
5451 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5452 CTLFLAG_RD, &txr->tso_tx,
5454 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5455 CTLFLAG_RD, &txr->no_tx_dma_setup,
5456 "Driver tx dma failure in xmit");
5457 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5458 CTLFLAG_RD, &txr->no_desc_avail,
5459 "Queue No Descriptor Available");
5460 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5461 CTLFLAG_RD, &txr->total_packets,
5462 "Queue Packets Transmitted");
5465 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5466 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5467 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5468 CTLFLAG_RD, NULL, "Queue Name");
5469 queue_list = SYSCTL_CHILDREN(queue_node);
5471 struct lro_ctrl *lro = &rxr->lro;
5473 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5474 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5475 CTLFLAG_RD, NULL, "Queue Name");
5476 queue_list = SYSCTL_CHILDREN(queue_node);
5478 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5479 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5480 ixgbe_sysctl_rdh_handler, "IU",
5481 "Receive Descriptor Head");
5482 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5483 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5484 ixgbe_sysctl_rdt_handler, "IU",
5485 "Receive Descriptor Tail");
5486 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5487 CTLFLAG_RD, &rxr->rx_packets,
5488 "Queue Packets Received");
5489 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5490 CTLFLAG_RD, &rxr->rx_bytes,
5491 "Queue Bytes Received");
5492 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5493 CTLFLAG_RD, &rxr->rx_copies,
5494 "Copied RX Frames");
5495 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5496 CTLFLAG_RD, &lro->lro_queued, 0,
5498 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5499 CTLFLAG_RD, &lro->lro_flushed, 0,
5503 /* MAC stats get the own sub node */
5505 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5506 CTLFLAG_RD, NULL, "MAC Statistics");
5507 stat_list = SYSCTL_CHILDREN(stat_node);
5509 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5510 CTLFLAG_RD, &stats->crcerrs,
5512 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5513 CTLFLAG_RD, &stats->illerrc,
5514 "Illegal Byte Errors");
5515 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5516 CTLFLAG_RD, &stats->errbc,
5518 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5519 CTLFLAG_RD, &stats->mspdc,
5520 "MAC Short Packets Discarded");
5521 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5522 CTLFLAG_RD, &stats->mlfc,
5523 "MAC Local Faults");
5524 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5525 CTLFLAG_RD, &stats->mrfc,
5526 "MAC Remote Faults");
5527 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5528 CTLFLAG_RD, &stats->rlec,
5529 "Receive Length Errors");
5531 /* Flow Control stats */
5532 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5533 CTLFLAG_RD, &stats->lxontxc,
5534 "Link XON Transmitted");
5535 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5536 CTLFLAG_RD, &stats->lxonrxc,
5537 "Link XON Received");
5538 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5539 CTLFLAG_RD, &stats->lxofftxc,
5540 "Link XOFF Transmitted");
5541 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5542 CTLFLAG_RD, &stats->lxoffrxc,
5543 "Link XOFF Received");
5545 /* Packet Reception Stats */
5546 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5547 CTLFLAG_RD, &stats->tor,
5548 "Total Octets Received");
5549 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5550 CTLFLAG_RD, &stats->gorc,
5551 "Good Octets Received");
5552 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5553 CTLFLAG_RD, &stats->tpr,
5554 "Total Packets Received");
5555 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5556 CTLFLAG_RD, &stats->gprc,
5557 "Good Packets Received");
5558 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5559 CTLFLAG_RD, &stats->mprc,
5560 "Multicast Packets Received");
5561 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5562 CTLFLAG_RD, &stats->bprc,
5563 "Broadcast Packets Received");
5564 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5565 CTLFLAG_RD, &stats->prc64,
5566 "64 byte frames received ");
5567 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5568 CTLFLAG_RD, &stats->prc127,
5569 "65-127 byte frames received");
5570 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5571 CTLFLAG_RD, &stats->prc255,
5572 "128-255 byte frames received");
5573 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5574 CTLFLAG_RD, &stats->prc511,
5575 "256-511 byte frames received");
5576 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5577 CTLFLAG_RD, &stats->prc1023,
5578 "512-1023 byte frames received");
5579 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5580 CTLFLAG_RD, &stats->prc1522,
5581 "1023-1522 byte frames received");
5582 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5583 CTLFLAG_RD, &stats->ruc,
5584 "Receive Undersized");
5585 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5586 CTLFLAG_RD, &stats->rfc,
5587 "Fragmented Packets Received ");
5588 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5589 CTLFLAG_RD, &stats->roc,
5590 "Oversized Packets Received");
5591 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5592 CTLFLAG_RD, &stats->rjc,
5594 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5595 CTLFLAG_RD, &stats->mngprc,
5596 "Management Packets Received");
5597 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5598 CTLFLAG_RD, &stats->mngptc,
5599 "Management Packets Dropped");
5600 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5601 CTLFLAG_RD, &stats->xec,
5604 /* Packet Transmission Stats */
5605 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5606 CTLFLAG_RD, &stats->gotc,
5607 "Good Octets Transmitted");
5608 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5609 CTLFLAG_RD, &stats->tpt,
5610 "Total Packets Transmitted");
5611 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5612 CTLFLAG_RD, &stats->gptc,
5613 "Good Packets Transmitted");
5614 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5615 CTLFLAG_RD, &stats->bptc,
5616 "Broadcast Packets Transmitted");
5617 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5618 CTLFLAG_RD, &stats->mptc,
5619 "Multicast Packets Transmitted");
5620 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5621 CTLFLAG_RD, &stats->mngptc,
5622 "Management Packets Transmitted");
5623 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5624 CTLFLAG_RD, &stats->ptc64,
5625 "64 byte frames transmitted ");
5626 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5627 CTLFLAG_RD, &stats->ptc127,
5628 "65-127 byte frames transmitted");
5629 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5630 CTLFLAG_RD, &stats->ptc255,
5631 "128-255 byte frames transmitted");
5632 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5633 CTLFLAG_RD, &stats->ptc511,
5634 "256-511 byte frames transmitted");
5635 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5636 CTLFLAG_RD, &stats->ptc1023,
5637 "512-1023 byte frames transmitted");
5638 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5639 CTLFLAG_RD, &stats->ptc1522,
5640 "1024-1522 byte frames transmitted");
5644 ** Set flow control using sysctl:
5645 ** Flow control values:
5652 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5655 struct adapter *adapter = (struct adapter *) arg1;
5658 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5659 if ((error) || (req->newptr == NULL))
5662 /* Don't bother if it's not changed */
5663 if (adapter->fc == last)
5666 switch (adapter->fc) {
5667 case ixgbe_fc_rx_pause:
5668 case ixgbe_fc_tx_pause:
5670 adapter->hw.fc.requested_mode = adapter->fc;
5671 if (adapter->num_queues > 1)
5672 ixgbe_disable_rx_drop(adapter);
5675 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5676 if (adapter->num_queues > 1)
5677 ixgbe_enable_rx_drop(adapter);
5683 /* Don't autoneg if forcing a value */
5684 adapter->hw.fc.disable_fc_autoneg = TRUE;
5685 ixgbe_fc_enable(&adapter->hw);
5690 ** Control link advertise speed:
5691 ** 1 - advertise only 1G
5692 ** 2 - advertise 100Mb
5693 ** 3 - advertise normal
5696 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5699 struct adapter *adapter;
5701 struct ixgbe_hw *hw;
5702 ixgbe_link_speed speed, last;
5704 adapter = (struct adapter *) arg1;
5707 last = adapter->advertise;
5709 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5710 if ((error) || (req->newptr == NULL))
5713 if (adapter->advertise == last) /* no change */
5716 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5717 (hw->phy.multispeed_fiber)))
5720 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5721 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5725 if (adapter->advertise == 1)
5726 speed = IXGBE_LINK_SPEED_1GB_FULL;
5727 else if (adapter->advertise == 2)
5728 speed = IXGBE_LINK_SPEED_100_FULL;
5729 else if (adapter->advertise == 3)
5730 speed = IXGBE_LINK_SPEED_1GB_FULL |
5731 IXGBE_LINK_SPEED_10GB_FULL;
5732 else { /* bogus value */
5733 adapter->advertise = last;
5737 hw->mac.autotry_restart = TRUE;
5738 hw->mac.ops.setup_link(hw, speed, TRUE);
5744 ** Thermal Shutdown Trigger
5745 ** - cause a Thermal Overtemp IRQ
5746 ** - this now requires firmware enabling
5749 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5751 int error, fire = 0;
5752 struct adapter *adapter = (struct adapter *) arg1;
5753 struct ixgbe_hw *hw = &adapter->hw;
5756 if (hw->mac.type != ixgbe_mac_X540)
5759 error = sysctl_handle_int(oidp, &fire, 0, req);
5760 if ((error) || (req->newptr == NULL))
5764 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5765 reg |= IXGBE_EICR_TS;
5766 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5773 ** Enable the hardware to drop packets when the buffer is
5774 ** full. This is useful when multiqueue,so that no single
5775 ** queue being full stalls the entire RX engine. We only
5776 ** enable this when Multiqueue AND when Flow Control is
5780 ixgbe_enable_rx_drop(struct adapter *adapter)
5782 struct ixgbe_hw *hw = &adapter->hw;
5784 for (int i = 0; i < adapter->num_queues; i++) {
5785 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5786 srrctl |= IXGBE_SRRCTL_DROP_EN;
5787 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5792 ixgbe_disable_rx_drop(struct adapter *adapter)
5794 struct ixgbe_hw *hw = &adapter->hw;
5796 for (int i = 0; i < adapter->num_queues; i++) {
5797 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5798 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5799 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);