1 /******************************************************************************
3 Copyright (c) 2013-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
35 #ifndef IXL_STANDALONE_BUILD
37 #include "opt_inet6.h"
45 #include <net/rss_config.h>
48 /*********************************************************************
50 *********************************************************************/
51 char ixl_driver_version[] = "1.4.27-k";
53 /*********************************************************************
56 * Used by probe to select devices to load on
57 * Last field stores an index into ixl_strings
58 * Last entry must be all 0s
60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61 *********************************************************************/
63 static ixl_vendor_info_t ixl_vendor_info_array[] =
65 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
66 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
67 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
68 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
69 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
70 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
71 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
72 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
73 /* required last entry */
77 /*********************************************************************
78 * Table of branding strings
79 *********************************************************************/
81 static char *ixl_strings[] = {
82 "Intel(R) Ethernet Connection XL710 Driver"
86 /*********************************************************************
88 *********************************************************************/
89 static int ixl_probe(device_t);
90 static int ixl_attach(device_t);
91 static int ixl_detach(device_t);
92 static int ixl_shutdown(device_t);
93 static int ixl_get_hw_capabilities(struct ixl_pf *);
94 static void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
95 static int ixl_ioctl(struct ifnet *, u_long, caddr_t);
96 static void ixl_init(void *);
97 static void ixl_init_locked(struct ixl_pf *);
98 static void ixl_stop(struct ixl_pf *);
99 static void ixl_stop_locked(struct ixl_pf *);
100 static void ixl_media_status(struct ifnet *, struct ifmediareq *);
101 static int ixl_media_change(struct ifnet *);
102 static void ixl_update_link_status(struct ixl_pf *);
103 static int ixl_allocate_pci_resources(struct ixl_pf *);
104 static u16 ixl_get_bus_info(struct i40e_hw *, device_t);
105 static int ixl_setup_stations(struct ixl_pf *);
106 static int ixl_switch_config(struct ixl_pf *);
107 static int ixl_initialize_vsi(struct ixl_vsi *);
109 static int ixl_setup_adminq_msix(struct ixl_pf *);
110 static int ixl_setup_adminq_tq(struct ixl_pf *);
111 static int ixl_setup_queue_msix(struct ixl_vsi *);
112 static int ixl_setup_queue_tqs(struct ixl_vsi *);
113 static int ixl_teardown_adminq_msix(struct ixl_pf *);
114 static int ixl_teardown_queue_msix(struct ixl_vsi *);
115 static void ixl_configure_intr0_msix(struct ixl_pf *);
116 static void ixl_configure_queue_intr_msix(struct ixl_pf *);
117 static void ixl_free_queue_tqs(struct ixl_vsi *);
118 static void ixl_free_adminq_tq(struct ixl_pf *);
120 static int ixl_assign_vsi_legacy(struct ixl_pf *);
121 static int ixl_init_msix(struct ixl_pf *);
122 static void ixl_configure_itr(struct ixl_pf *);
123 static void ixl_configure_legacy(struct ixl_pf *);
124 static void ixl_free_pci_resources(struct ixl_pf *);
125 static void ixl_local_timer(void *);
126 static int ixl_setup_interface(device_t, struct ixl_vsi *);
127 static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
128 static void ixl_config_rss(struct ixl_vsi *);
129 static void ixl_set_queue_rx_itr(struct ixl_queue *);
130 static void ixl_set_queue_tx_itr(struct ixl_queue *);
131 static int ixl_set_advertised_speeds(struct ixl_pf *, int);
132 static void ixl_get_initial_advertised_speeds(struct ixl_pf *);
134 static int ixl_enable_rings(struct ixl_vsi *);
135 static int ixl_disable_rings(struct ixl_vsi *);
136 static void ixl_enable_intr(struct ixl_vsi *);
137 static void ixl_disable_intr(struct ixl_vsi *);
138 static void ixl_disable_rings_intr(struct ixl_vsi *);
140 static void ixl_enable_adminq(struct i40e_hw *);
141 static void ixl_disable_adminq(struct i40e_hw *);
142 static void ixl_enable_queue(struct i40e_hw *, int);
143 static void ixl_disable_queue(struct i40e_hw *, int);
144 static void ixl_enable_legacy(struct i40e_hw *);
145 static void ixl_disable_legacy(struct i40e_hw *);
147 static void ixl_set_promisc(struct ixl_vsi *);
148 static void ixl_add_multi(struct ixl_vsi *);
149 static void ixl_del_multi(struct ixl_vsi *);
150 static void ixl_register_vlan(void *, struct ifnet *, u16);
151 static void ixl_unregister_vlan(void *, struct ifnet *, u16);
152 static void ixl_setup_vlan_filters(struct ixl_vsi *);
154 static void ixl_init_filters(struct ixl_vsi *);
155 static void ixl_reconfigure_filters(struct ixl_vsi *vsi);
156 static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
157 static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
158 static void ixl_add_hw_filters(struct ixl_vsi *, int, int);
159 static void ixl_del_hw_filters(struct ixl_vsi *, int);
160 static struct ixl_mac_filter *
161 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
162 static void ixl_add_mc_filter(struct ixl_vsi *, u8 *);
163 static void ixl_free_mac_filters(struct ixl_vsi *vsi);
166 static void ixl_add_device_sysctls(struct ixl_pf *);
168 static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
169 static int ixl_set_advertise(SYSCTL_HANDLER_ARGS);
170 static int ixl_current_speed(SYSCTL_HANDLER_ARGS);
171 static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
173 #ifdef IXL_DEBUG_SYSCTL
174 static int ixl_debug_info(SYSCTL_HANDLER_ARGS);
175 static void ixl_print_debug_info(struct ixl_pf *);
177 static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
178 static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
179 static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
180 static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
181 static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
184 /* The MSI/X Interrupt handlers */
185 static void ixl_intr(void *);
186 static void ixl_msix_que(void *);
187 static void ixl_msix_adminq(void *);
188 static void ixl_handle_mdd_event(struct ixl_pf *);
190 /* Deferred interrupt tasklets */
191 static void ixl_do_adminq(void *, int);
194 static void ixl_add_hw_stats(struct ixl_pf *);
195 static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
196 struct sysctl_oid_list *, struct i40e_hw_port_stats *);
197 static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
198 struct sysctl_oid_list *,
199 struct i40e_eth_stats *);
200 static void ixl_update_stats_counters(struct ixl_pf *);
201 static void ixl_update_eth_stats(struct ixl_vsi *);
202 static void ixl_update_vsi_stats(struct ixl_vsi *);
203 static void ixl_pf_reset_stats(struct ixl_pf *);
204 static void ixl_vsi_reset_stats(struct ixl_vsi *);
205 static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
207 static void ixl_stat_update32(struct i40e_hw *, u32, bool,
210 static int ixl_handle_nvmupd_cmd(struct ixl_pf *, struct ifdrv *);
211 static void ixl_handle_empr_reset(struct ixl_pf *);
212 static int ixl_rebuild_hw_structs_after_reset(struct ixl_pf *);
214 /* Debug helper functions */
216 static void ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *);
220 static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
222 static int ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
223 static void ixl_iov_uninit(device_t dev);
224 static int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
226 static void ixl_handle_vf_msg(struct ixl_pf *,
227 struct i40e_arq_event_info *);
228 static void ixl_handle_vflr(void *arg, int pending);
230 static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
231 static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
234 /*********************************************************************
235 * FreeBSD Device Interface Entry Points
236 *********************************************************************/
238 static device_method_t ixl_methods[] = {
239 /* Device interface */
240 DEVMETHOD(device_probe, ixl_probe),
241 DEVMETHOD(device_attach, ixl_attach),
242 DEVMETHOD(device_detach, ixl_detach),
243 DEVMETHOD(device_shutdown, ixl_shutdown),
245 DEVMETHOD(pci_iov_init, ixl_iov_init),
246 DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
247 DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
252 static driver_t ixl_driver = {
253 "ixl", ixl_methods, sizeof(struct ixl_pf),
256 devclass_t ixl_devclass;
257 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
259 MODULE_DEPEND(ixl, pci, 1, 1, 1);
260 MODULE_DEPEND(ixl, ether, 1, 1, 1);
262 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
263 #endif /* DEV_NETMAP */
266 ** Global reset mutex
268 static struct mtx ixl_reset_mtx;
271 ** TUNEABLE PARAMETERS:
274 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
275 "IXL driver parameters");
278 * MSIX should be the default for best performance,
279 * but this allows it to be forced off for testing.
281 static int ixl_enable_msix = 1;
282 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
283 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
284 "Enable MSI-X interrupts");
287 ** Number of descriptors per ring:
288 ** - TX and RX are the same size
290 static int ixl_ringsz = DEFAULT_RING;
291 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
292 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
293 &ixl_ringsz, 0, "Descriptor Ring Size");
296 ** This can be set manually, if left as 0 the
297 ** number of queues will be calculated based
298 ** on cpus and msix vectors available.
300 int ixl_max_queues = 0;
301 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
302 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
303 &ixl_max_queues, 0, "Number of Queues");
306 ** Controls for Interrupt Throttling
307 ** - true/false for dynamic adjustment
308 ** - default values for static ITR
310 int ixl_dynamic_rx_itr = 1;
311 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
312 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
313 &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
315 int ixl_dynamic_tx_itr = 1;
316 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
317 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
318 &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
320 int ixl_rx_itr = IXL_ITR_8K;
321 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
322 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
323 &ixl_rx_itr, 0, "RX Interrupt Rate");
325 int ixl_tx_itr = IXL_ITR_4K;
326 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
327 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
328 &ixl_tx_itr, 0, "TX Interrupt Rate");
331 static int ixl_enable_fdir = 1;
332 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
333 /* Rate at which we sample */
334 int ixl_atr_rate = 20;
335 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
339 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
340 #include <dev/netmap/if_ixl_netmap.h>
341 #endif /* DEV_NETMAP */
343 static char *ixl_fc_string[6] = {
352 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
354 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
355 {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
357 /*********************************************************************
358 * Device identification routine
360 * ixl_probe determines if the driver should be loaded on
361 * the hardware based on PCI vendor/device id of the device.
363 * return BUS_PROBE_DEFAULT on success, positive on failure
364 *********************************************************************/
367 ixl_probe(device_t dev)
369 ixl_vendor_info_t *ent;
371 u16 pci_vendor_id, pci_device_id;
372 u16 pci_subvendor_id, pci_subdevice_id;
373 char device_name[256];
374 static bool lock_init = FALSE;
377 INIT_DEBUGOUT("ixl_probe: begin");
379 pci_vendor_id = pci_get_vendor(dev);
380 if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
383 pci_device_id = pci_get_device(dev);
384 pci_subvendor_id = pci_get_subvendor(dev);
385 pci_subdevice_id = pci_get_subdevice(dev);
387 ent = ixl_vendor_info_array;
388 while (ent->vendor_id != 0) {
389 if ((pci_vendor_id == ent->vendor_id) &&
390 (pci_device_id == ent->device_id) &&
392 ((pci_subvendor_id == ent->subvendor_id) ||
393 (ent->subvendor_id == 0)) &&
395 ((pci_subdevice_id == ent->subdevice_id) ||
396 (ent->subdevice_id == 0))) {
397 sprintf(device_name, "%s, Version - %s",
398 ixl_strings[ent->index],
400 device_set_desc_copy(dev, device_name);
401 /* One shot mutex init */
402 if (lock_init == FALSE) {
404 mtx_init(&ixl_reset_mtx,
406 "IXL RESET Lock", MTX_DEF);
408 return (BUS_PROBE_DEFAULT);
415 /*********************************************************************
416 * Device initialization routine
418 * The attach entry point is called when the driver is being loaded.
419 * This routine identifies the type of hardware, allocates all resources
420 * and initializes the hardware.
422 * return 0 on success, positive on failure
423 *********************************************************************/
426 ixl_attach(device_t dev)
434 nvlist_t *pf_schema, *vf_schema;
438 INIT_DEBUGOUT("ixl_attach: begin");
440 /* Allocate, clear, and link in our primary soft structure */
441 pf = device_get_softc(dev);
442 pf->dev = pf->osdep.dev = dev;
446 ** Note this assumes we have a single embedded VSI,
447 ** this could be enhanced later to allocate multiple
453 IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
455 /* Set up the timer callout */
456 callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
458 /* Save off the PCI information */
459 hw->vendor_id = pci_get_vendor(dev);
460 hw->device_id = pci_get_device(dev);
461 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
462 hw->subsystem_vendor_id =
463 pci_read_config(dev, PCIR_SUBVEND_0, 2);
464 hw->subsystem_device_id =
465 pci_read_config(dev, PCIR_SUBDEV_0, 2);
467 hw->bus.device = pci_get_slot(dev);
468 hw->bus.func = pci_get_function(dev);
470 pf->vc_debug_lvl = 1;
472 /* Do PCI setup - map BAR0, etc */
473 if (ixl_allocate_pci_resources(pf)) {
474 device_printf(dev, "Allocation of PCI resources failed\n");
479 /* Establish a clean starting point */
481 error = i40e_pf_reset(hw);
483 device_printf(dev, "PF reset failure %d\n", error);
488 /* Set admin queue parameters */
489 hw->aq.num_arq_entries = IXL_AQ_LEN;
490 hw->aq.num_asq_entries = IXL_AQ_LEN;
491 hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
492 hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
494 /* Initialize mac filter list for VSI */
495 SLIST_INIT(&vsi->ftl);
497 /* Initialize the shared code */
498 error = i40e_init_shared_code(hw);
500 device_printf(dev, "Unable to initialize shared code, error %d\n",
506 /* Set up the admin queue */
507 error = i40e_init_adminq(hw);
508 if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
509 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
514 ixl_print_nvm_version(pf);
516 if (error == I40E_ERR_FIRMWARE_API_VERSION) {
517 device_printf(dev, "The driver for the device stopped "
518 "because the NVM image is newer than expected.\n"
519 "You must install the most recent version of "
520 "the network driver.\n");
525 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
526 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
527 device_printf(dev, "The driver for the device detected "
528 "a newer version of the NVM image than expected.\n"
529 "Please install the most recent version of the network driver.\n");
530 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
531 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
532 device_printf(dev, "The driver for the device detected "
533 "an older version of the NVM image than expected.\n"
534 "Please update the NVM image.\n");
537 i40e_clear_pxe_mode(hw);
539 /* Get capabilities from the device */
540 error = ixl_get_hw_capabilities(pf);
542 device_printf(dev, "HW capabilities failure!\n");
546 /* Set up host memory cache */
547 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
548 hw->func_caps.num_rx_qp, 0, 0);
550 device_printf(dev, "init_lan_hmc failed: %d\n", error);
554 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
556 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
560 /* Disable LLDP from the firmware for certain NVM versions */
561 if (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
562 (pf->hw.aq.fw_maj_ver < 4))
563 i40e_aq_stop_lldp(hw, TRUE, NULL);
565 i40e_get_mac_addr(hw, hw->mac.addr);
566 error = i40e_validate_mac_addr(hw->mac.addr);
568 device_printf(dev, "validate_mac_addr failed: %d\n", error);
571 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
572 i40e_get_port_mac_addr(hw, hw->mac.port_addr);
574 /* Set up VSI and queues */
575 if (ixl_setup_stations(pf) != 0) {
576 device_printf(dev, "setup stations failed!\n");
581 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
582 (hw->aq.fw_maj_ver < 4)) {
584 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
586 device_printf(dev, "link restart failed, aq_err=%d\n",
587 pf->hw.aq.asq_last_status);
592 /* Determine link state */
593 hw->phy.get_link_info = TRUE;
594 i40e_get_link_status(hw, &pf->link_up);
596 /* Setup OS network interface / ifnet */
597 if (ixl_setup_interface(dev, vsi) != 0) {
598 device_printf(dev, "interface setup failed!\n");
603 error = ixl_switch_config(pf);
605 device_printf(dev, "Initial ixl_switch_config() failed: %d\n",
610 /* Limit PHY interrupts to link, autoneg, and modules failure */
611 error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
614 device_printf(dev, "i40e_aq_set_phy_mask() failed: err %d,"
615 " aq_err %d\n", error, hw->aq.asq_last_status);
619 /* Get the bus configuration and set the shared code's config */
620 bus = ixl_get_bus_info(hw, dev);
621 i40e_set_pci_config_data(hw, bus);
624 * In MSI-X mode, initialize the Admin Queue interrupt,
625 * so userland tools can communicate with the adapter regardless of
626 * the ifnet interface's status.
629 error = ixl_setup_adminq_msix(pf);
631 device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
635 error = ixl_setup_adminq_tq(pf);
637 device_printf(dev, "ixl_setup_adminq_tq error: %d\n",
641 ixl_configure_intr0_msix(pf);
642 ixl_enable_adminq(hw);
645 /* Initialize statistics & add sysctls */
646 ixl_add_device_sysctls(pf);
648 ixl_pf_reset_stats(pf);
649 ixl_update_stats_counters(pf);
650 ixl_add_hw_stats(pf);
652 /* Register for VLAN events */
653 vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
654 ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
655 vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
656 ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
659 /* SR-IOV is only supported when MSI-X is in use. */
661 pf_schema = pci_iov_schema_alloc_node();
662 vf_schema = pci_iov_schema_alloc_node();
663 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
664 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
665 IOV_SCHEMA_HASDEFAULT, TRUE);
666 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
667 IOV_SCHEMA_HASDEFAULT, FALSE);
668 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
669 IOV_SCHEMA_HASDEFAULT, FALSE);
671 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
672 if (iov_error != 0) {
674 "Failed to initialize SR-IOV (error=%d)\n",
677 device_printf(dev, "SR-IOV ready\n");
682 ixl_netmap_attach(vsi);
683 #endif /* DEV_NETMAP */
684 INIT_DEBUGOUT("ixl_attach: end");
688 if (vsi->ifp != NULL)
691 i40e_shutdown_lan_hmc(hw);
693 i40e_shutdown_adminq(hw);
695 ixl_free_pci_resources(pf);
697 IXL_PF_LOCK_DESTROY(pf);
701 /*********************************************************************
702 * Device removal routine
704 * The detach entry point is called when the driver is being removed.
705 * This routine stops the adapter and deallocates all the resources
706 * that were allocated for driver operation.
708 * return 0 on success, positive on failure
709 *********************************************************************/
712 ixl_detach(device_t dev)
714 struct ixl_pf *pf = device_get_softc(dev);
715 struct i40e_hw *hw = &pf->hw;
716 struct ixl_vsi *vsi = &pf->vsi;
717 enum i40e_status_code status;
722 INIT_DEBUGOUT("ixl_detach: begin");
724 /* Make sure VLANS are not using driver */
725 if (vsi->ifp->if_vlantrunk != NULL) {
726 device_printf(dev, "Vlan in use, detach first\n");
731 error = pci_iov_detach(dev);
733 device_printf(dev, "SR-IOV in use; detach first.\n");
738 ether_ifdetach(vsi->ifp);
739 if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)
742 ixl_free_queue_tqs(vsi);
744 /* Shutdown LAN HMC */
745 status = i40e_shutdown_lan_hmc(hw);
748 "Shutdown LAN HMC failed with code %d\n", status);
750 /* Shutdown admin queue */
751 ixl_disable_adminq(hw);
752 ixl_free_adminq_tq(pf);
753 ixl_teardown_adminq_msix(pf);
754 status = i40e_shutdown_adminq(hw);
757 "Shutdown Admin queue failed with code %d\n", status);
759 /* Unregister VLAN events */
760 if (vsi->vlan_attach != NULL)
761 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
762 if (vsi->vlan_detach != NULL)
763 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
765 callout_drain(&pf->timer);
767 netmap_detach(vsi->ifp);
768 #endif /* DEV_NETMAP */
769 ixl_free_pci_resources(pf);
770 bus_generic_detach(dev);
773 IXL_PF_LOCK_DESTROY(pf);
777 /*********************************************************************
779 * Shutdown entry point
781 **********************************************************************/
784 ixl_shutdown(device_t dev)
786 struct ixl_pf *pf = device_get_softc(dev);
792 /*********************************************************************
794 * Get the hardware capabilities
796 **********************************************************************/
799 ixl_get_hw_capabilities(struct ixl_pf *pf)
801 struct i40e_aqc_list_capabilities_element_resp *buf;
802 struct i40e_hw *hw = &pf->hw;
803 device_t dev = pf->dev;
808 len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
810 if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
811 malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
812 device_printf(dev, "Unable to allocate cap memory\n");
816 /* This populates the hw struct */
817 error = i40e_aq_discover_capabilities(hw, buf, len,
818 &needed, i40e_aqc_opc_list_func_capabilities, NULL);
820 if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
822 /* retry once with a larger buffer */
826 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
827 device_printf(dev, "capability discovery failed: %d\n",
828 pf->hw.aq.asq_last_status);
832 /* Capture this PF's starting queue pair */
833 pf->qbase = hw->func_caps.base_queue;
836 device_printf(dev, "pf_id=%d, num_vfs=%d, msix_pf=%d, "
837 "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
838 hw->pf_id, hw->func_caps.num_vfs,
839 hw->func_caps.num_msix_vectors,
840 hw->func_caps.num_msix_vectors_vf,
841 hw->func_caps.fd_filters_guaranteed,
842 hw->func_caps.fd_filters_best_effort,
843 hw->func_caps.num_tx_qp,
844 hw->func_caps.num_rx_qp,
845 hw->func_caps.base_queue);
851 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
853 device_t dev = vsi->dev;
855 /* Enable/disable TXCSUM/TSO4 */
856 if (!(ifp->if_capenable & IFCAP_TXCSUM)
857 && !(ifp->if_capenable & IFCAP_TSO4)) {
858 if (mask & IFCAP_TXCSUM) {
859 ifp->if_capenable |= IFCAP_TXCSUM;
860 /* enable TXCSUM, restore TSO if previously enabled */
861 if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
862 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
863 ifp->if_capenable |= IFCAP_TSO4;
866 else if (mask & IFCAP_TSO4) {
867 ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
868 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
870 "TSO4 requires txcsum, enabling both...\n");
872 } else if((ifp->if_capenable & IFCAP_TXCSUM)
873 && !(ifp->if_capenable & IFCAP_TSO4)) {
874 if (mask & IFCAP_TXCSUM)
875 ifp->if_capenable &= ~IFCAP_TXCSUM;
876 else if (mask & IFCAP_TSO4)
877 ifp->if_capenable |= IFCAP_TSO4;
878 } else if((ifp->if_capenable & IFCAP_TXCSUM)
879 && (ifp->if_capenable & IFCAP_TSO4)) {
880 if (mask & IFCAP_TXCSUM) {
881 vsi->flags |= IXL_FLAGS_KEEP_TSO4;
882 ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
884 "TSO4 requires txcsum, disabling both...\n");
885 } else if (mask & IFCAP_TSO4)
886 ifp->if_capenable &= ~IFCAP_TSO4;
889 /* Enable/disable TXCSUM_IPV6/TSO6 */
890 if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
891 && !(ifp->if_capenable & IFCAP_TSO6)) {
892 if (mask & IFCAP_TXCSUM_IPV6) {
893 ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
894 if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
895 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
896 ifp->if_capenable |= IFCAP_TSO6;
898 } else if (mask & IFCAP_TSO6) {
899 ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
900 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
902 "TSO6 requires txcsum6, enabling both...\n");
904 } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
905 && !(ifp->if_capenable & IFCAP_TSO6)) {
906 if (mask & IFCAP_TXCSUM_IPV6)
907 ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
908 else if (mask & IFCAP_TSO6)
909 ifp->if_capenable |= IFCAP_TSO6;
910 } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
911 && (ifp->if_capenable & IFCAP_TSO6)) {
912 if (mask & IFCAP_TXCSUM_IPV6) {
913 vsi->flags |= IXL_FLAGS_KEEP_TSO6;
914 ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
916 "TSO6 requires txcsum6, disabling both...\n");
917 } else if (mask & IFCAP_TSO6)
918 ifp->if_capenable &= ~IFCAP_TSO6;
922 /*********************************************************************
925 * ixl_ioctl is called when the user wants to configure the
928 * return 0 on success, positive on failure
929 **********************************************************************/
932 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
934 struct ixl_vsi *vsi = ifp->if_softc;
935 struct ixl_pf *pf = vsi->back;
936 struct ifreq *ifr = (struct ifreq *)data;
937 struct ifdrv *ifd = (struct ifdrv *)data;
938 #if defined(INET) || defined(INET6)
939 struct ifaddr *ifa = (struct ifaddr *)data;
940 bool avoid_reset = FALSE;
948 if (ifa->ifa_addr->sa_family == AF_INET)
952 if (ifa->ifa_addr->sa_family == AF_INET6)
955 #if defined(INET) || defined(INET6)
957 ** Calling init results in link renegotiation,
958 ** so we avoid doing it when possible.
961 ifp->if_flags |= IFF_UP;
962 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
965 if (!(ifp->if_flags & IFF_NOARP))
966 arp_ifinit(ifp, ifa);
969 error = ether_ioctl(ifp, command, data);
973 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
974 if (ifr->ifr_mtu > IXL_MAX_FRAME -
975 ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
979 ifp->if_mtu = ifr->ifr_mtu;
980 vsi->max_frame_size =
981 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
982 + ETHER_VLAN_ENCAP_LEN;
988 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
990 if (ifp->if_flags & IFF_UP) {
991 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
992 if ((ifp->if_flags ^ pf->if_flags) &
993 (IFF_PROMISC | IFF_ALLMULTI)) {
994 ixl_set_promisc(vsi);
1002 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1008 pf->if_flags = ifp->if_flags;
1013 IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific "
1016 /* NVM update command */
1017 if (ifd->ifd_cmd == I40E_NVM_ACCESS)
1018 error = ixl_handle_nvmupd_cmd(pf, ifd);
1023 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1024 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1026 ixl_disable_intr(vsi);
1028 ixl_enable_intr(vsi);
1033 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1034 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1036 ixl_disable_intr(vsi);
1038 ixl_enable_intr(vsi);
1044 #ifdef IFM_ETH_XTYPE
1047 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1048 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1052 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1053 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1055 ixl_cap_txcsum_tso(vsi, ifp, mask);
1057 if (mask & IFCAP_RXCSUM)
1058 ifp->if_capenable ^= IFCAP_RXCSUM;
1059 if (mask & IFCAP_RXCSUM_IPV6)
1060 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1061 if (mask & IFCAP_LRO)
1062 ifp->if_capenable ^= IFCAP_LRO;
1063 if (mask & IFCAP_VLAN_HWTAGGING)
1064 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1065 if (mask & IFCAP_VLAN_HWFILTER)
1066 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1067 if (mask & IFCAP_VLAN_HWTSO)
1068 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1069 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1071 ixl_init_locked(pf);
1074 VLAN_CAPABILITIES(ifp);
1080 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1081 error = ether_ioctl(ifp, command, data);
1089 /*********************************************************************
1092 * This routine is used in two ways. It is used by the stack as
1093 * init entry point in network interface structure. It is also used
1094 * by the driver as a hw/sw initialization routine to get to a
1097 * return 0 on success, positive on failure
1098 **********************************************************************/
1101 ixl_init_locked(struct ixl_pf *pf)
1103 struct i40e_hw *hw = &pf->hw;
1104 struct ixl_vsi *vsi = &pf->vsi;
1105 struct ifnet *ifp = vsi->ifp;
1106 device_t dev = pf->dev;
1107 struct i40e_filter_control_settings filter;
1108 u8 tmpaddr[ETHER_ADDR_LEN];
1111 mtx_assert(&pf->pf_mtx, MA_OWNED);
1112 INIT_DEBUGOUT("ixl_init_locked: begin");
1114 ixl_stop_locked(pf);
1116 /* Get the latest mac address... User might use a LAA */
1117 bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1118 I40E_ETH_LENGTH_OF_ADDRESS);
1119 if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1120 (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
1121 ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1122 bcopy(tmpaddr, hw->mac.addr,
1123 I40E_ETH_LENGTH_OF_ADDRESS);
1124 ret = i40e_aq_mac_address_write(hw,
1125 I40E_AQC_WRITE_TYPE_LAA_ONLY,
1126 hw->mac.addr, NULL);
1128 device_printf(dev, "LLA address"
1129 "change failed!!\n");
1134 ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1136 /* Set the various hardware offload abilities */
1137 ifp->if_hwassist = 0;
1138 if (ifp->if_capenable & IFCAP_TSO)
1139 ifp->if_hwassist |= CSUM_TSO;
1140 if (ifp->if_capenable & IFCAP_TXCSUM)
1141 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1142 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1143 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1145 /* Set up the device filtering */
1146 bzero(&filter, sizeof(filter));
1147 filter.enable_ethtype = TRUE;
1148 filter.enable_macvlan = TRUE;
1150 filter.enable_fdir = TRUE;
1152 filter.hash_lut_size = I40E_HASH_LUT_SIZE_512;
1153 if (i40e_set_filter_control(hw, &filter))
1154 device_printf(dev, "i40e_set_filter_control() failed\n");
1157 ixl_config_rss(vsi);
1159 /* Prepare the VSI: rings, hmc contexts, etc... */
1160 if (ixl_initialize_vsi(vsi)) {
1161 device_printf(dev, "initialize vsi failed!!\n");
1165 /* Add protocol filters to list */
1166 ixl_init_filters(vsi);
1168 /* Setup vlan's if needed */
1169 ixl_setup_vlan_filters(vsi);
1171 /* Set up MSI/X routing and the ITR settings */
1172 if (ixl_enable_msix) {
1173 ixl_configure_queue_intr_msix(pf);
1174 ixl_configure_itr(pf);
1176 ixl_configure_legacy(pf);
1178 ixl_enable_rings(vsi);
1180 i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1182 ixl_reconfigure_filters(vsi);
1184 /* And now turn on interrupts */
1185 ixl_enable_intr(vsi);
1188 hw->phy.get_link_info = TRUE;
1189 i40e_get_link_status(hw, &pf->link_up);
1190 ixl_update_link_status(pf);
1192 /* Set initial advertised speed sysctl value */
1193 ixl_get_initial_advertised_speeds(pf);
1195 /* Start the local timer */
1196 callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1198 /* Now inform the stack we're ready */
1199 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1204 /* For the set_advertise sysctl */
1206 ixl_get_initial_advertised_speeds(struct ixl_pf *pf)
1208 struct i40e_hw *hw = &pf->hw;
1209 device_t dev = pf->dev;
1210 enum i40e_status_code status;
1211 struct i40e_aq_get_phy_abilities_resp abilities;
1213 /* Set initial sysctl values */
1214 status = i40e_aq_get_phy_capabilities(hw, FALSE, false, &abilities,
1217 /* Non-fatal error */
1218 device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n",
1223 if (abilities.link_speed & I40E_LINK_SPEED_40GB)
1224 pf->advertised_speed |= 0x10;
1225 if (abilities.link_speed & I40E_LINK_SPEED_20GB)
1226 pf->advertised_speed |= 0x8;
1227 if (abilities.link_speed & I40E_LINK_SPEED_10GB)
1228 pf->advertised_speed |= 0x4;
1229 if (abilities.link_speed & I40E_LINK_SPEED_1GB)
1230 pf->advertised_speed |= 0x2;
1231 if (abilities.link_speed & I40E_LINK_SPEED_100MB)
1232 pf->advertised_speed |= 0x1;
1236 ixl_teardown_hw_structs(struct ixl_pf *pf)
1238 enum i40e_status_code status = 0;
1239 struct i40e_hw *hw = &pf->hw;
1240 device_t dev = pf->dev;
1242 /* Shutdown LAN HMC */
1243 if (hw->hmc.hmc_obj) {
1244 status = i40e_shutdown_lan_hmc(hw);
1247 "init: LAN HMC shutdown failure; status %d\n", status);
1252 // XXX: This gets called when we know the adminq is inactive;
1253 // so we already know it's setup when we get here.
1255 /* Shutdown admin queue */
1256 status = i40e_shutdown_adminq(hw);
1259 "init: Admin Queue shutdown failure; status %d\n", status);
1266 ixl_reset(struct ixl_pf *pf)
1268 struct i40e_hw *hw = &pf->hw;
1269 device_t dev = pf->dev;
1273 // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary
1275 error = i40e_pf_reset(hw);
1277 device_printf(dev, "init: PF reset failure");
1282 error = i40e_init_adminq(hw);
1284 device_printf(dev, "init: Admin queue init failure;"
1285 " status code %d", error);
1290 i40e_clear_pxe_mode(hw);
1292 error = ixl_get_hw_capabilities(pf);
1294 device_printf(dev, "init: Error retrieving HW capabilities;"
1295 " status code %d\n", error);
1299 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1300 hw->func_caps.num_rx_qp, 0, 0);
1302 device_printf(dev, "init: LAN HMC init failed; status code %d\n",
1308 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1310 device_printf(dev, "init: LAN HMC config failed; status code %d\n",
1316 // XXX: possible fix for panic, but our failure recovery is still broken
1317 error = ixl_switch_config(pf);
1319 device_printf(dev, "init: ixl_switch_config() failed: %d\n",
1324 error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
1327 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
1328 " aq_err %d\n", error, hw->aq.asq_last_status);
1333 error = i40e_set_fc(hw, &set_fc_err_mask, true);
1335 device_printf(dev, "init: setting link flow control failed; retcode %d,"
1336 " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
1340 // XXX: (Rebuild VSIs?)
1342 /* Firmware delay workaround */
1343 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
1344 (hw->aq.fw_maj_ver < 4)) {
1345 i40e_msec_delay(75);
1346 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
1348 device_printf(dev, "init: link restart failed, aq_err %d\n",
1349 hw->aq.asq_last_status);
1362 struct ixl_pf *pf = arg;
1363 struct ixl_vsi *vsi = &pf->vsi;
1364 device_t dev = pf->dev;
1368 * If the aq is dead here, it probably means something outside of the driver
1369 * did something to the adapter, like a PF reset.
1370 * So rebuild the driver's state here if that occurs.
1372 if (!i40e_check_asq_alive(&pf->hw)) {
1373 device_printf(dev, "Admin Queue is down; resetting...\n");
1375 ixl_teardown_hw_structs(pf);
1381 * Set up LAN queue interrupts here.
1382 * Kernel interrupt setup functions cannot be called while holding a lock,
1383 * so this is done outside of init_locked().
1386 /* Teardown existing interrupts, if they exist */
1387 ixl_teardown_queue_msix(vsi);
1388 ixl_free_queue_tqs(vsi);
1389 /* Then set them up again */
1390 error = ixl_setup_queue_msix(vsi);
1392 device_printf(dev, "ixl_setup_queue_msix() error: %d\n",
1394 error = ixl_setup_queue_tqs(vsi);
1396 device_printf(dev, "ixl_setup_queue_tqs() error: %d\n",
1400 error = ixl_assign_vsi_legacy(pf);
1402 device_printf(pf->dev, "assign_vsi_msix/legacy error: %d\n", error);
1407 ixl_init_locked(pf);
1412 ** MSIX Interrupt Handlers and Tasklets
1415 ixl_handle_que(void *context, int pending)
1417 struct ixl_queue *que = context;
1418 struct ixl_vsi *vsi = que->vsi;
1419 struct i40e_hw *hw = vsi->hw;
1420 struct tx_ring *txr = &que->txr;
1421 struct ifnet *ifp = vsi->ifp;
1424 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1425 more = ixl_rxeof(que, IXL_RX_LIMIT);
1428 if (!drbr_empty(ifp, txr->br))
1429 ixl_mq_start_locked(ifp, txr);
1432 taskqueue_enqueue(que->tq, &que->task);
1437 /* Reenable this interrupt - hmmm */
1438 ixl_enable_queue(hw, que->me);
1443 /*********************************************************************
1445 * Legacy Interrupt Service routine
1447 **********************************************************************/
1451 struct ixl_pf *pf = arg;
1452 struct i40e_hw *hw = &pf->hw;
1453 struct ixl_vsi *vsi = &pf->vsi;
1454 struct ixl_queue *que = vsi->queues;
1455 struct ifnet *ifp = vsi->ifp;
1456 struct tx_ring *txr = &que->txr;
1457 u32 reg, icr0, mask;
1458 bool more_tx, more_rx;
1462 /* Protect against spurious interrupts */
1463 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1466 icr0 = rd32(hw, I40E_PFINT_ICR0);
1468 reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1469 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1470 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1472 mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1475 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1476 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1479 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1480 taskqueue_enqueue(pf->tq, &pf->adminq);
1484 more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1487 more_tx = ixl_txeof(que);
1488 if (!drbr_empty(vsi->ifp, txr->br))
1492 /* re-enable other interrupt causes */
1493 wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1495 /* And now the queues */
1496 reg = rd32(hw, I40E_QINT_RQCTL(0));
1497 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1498 wr32(hw, I40E_QINT_RQCTL(0), reg);
1500 reg = rd32(hw, I40E_QINT_TQCTL(0));
1501 reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1502 reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1503 wr32(hw, I40E_QINT_TQCTL(0), reg);
1505 ixl_enable_legacy(hw);
1511 /*********************************************************************
1513 * MSIX VSI Interrupt Service routine
1515 **********************************************************************/
1517 ixl_msix_que(void *arg)
1519 struct ixl_queue *que = arg;
1520 struct ixl_vsi *vsi = que->vsi;
1521 struct i40e_hw *hw = vsi->hw;
1522 struct tx_ring *txr = &que->txr;
1523 bool more_tx, more_rx;
1525 /* Protect against spurious interrupts */
1526 if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1531 more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1534 more_tx = ixl_txeof(que);
1536 ** Make certain that if the stack
1537 ** has anything queued the task gets
1538 ** scheduled to handle it.
1540 if (!drbr_empty(vsi->ifp, txr->br))
1544 ixl_set_queue_rx_itr(que);
1545 ixl_set_queue_tx_itr(que);
1547 if (more_tx || more_rx)
1548 taskqueue_enqueue(que->tq, &que->task);
1550 ixl_enable_queue(hw, que->me);
1556 /*********************************************************************
1558 * MSIX Admin Queue Interrupt Service routine
1560 **********************************************************************/
1562 ixl_msix_adminq(void *arg)
1564 struct ixl_pf *pf = arg;
1565 struct i40e_hw *hw = &pf->hw;
1566 u32 reg, mask, rstat_reg;
1567 bool do_task = FALSE;
1571 reg = rd32(hw, I40E_PFINT_ICR0);
1572 mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1574 /* Check on the cause */
1575 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) {
1576 mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK;
1580 if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1581 ixl_handle_mdd_event(pf);
1582 mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK;
1585 if (reg & I40E_PFINT_ICR0_GRST_MASK) {
1586 device_printf(pf->dev, "Reset Requested!\n");
1587 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT);
1588 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
1589 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
1590 device_printf(pf->dev, "Reset type: ");
1591 switch (rstat_reg) {
1592 /* These others might be handled similarly to an EMPR reset */
1593 case I40E_RESET_CORER:
1596 case I40E_RESET_GLOBR:
1599 case I40E_RESET_EMPR:
1601 atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
1607 // overload admin queue task to check reset progress?
1611 if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) {
1612 device_printf(pf->dev, "ECC Error detected!\n");
1615 if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) {
1616 device_printf(pf->dev, "HMC Error detected!\n");
1619 if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) {
1620 device_printf(pf->dev, "PCI Exception detected!\n");
1624 if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1625 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1626 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1630 reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1631 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1632 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1635 taskqueue_enqueue(pf->tq, &pf->adminq);
1638 /*********************************************************************
1640 * Media Ioctl callback
1642 * This routine is called whenever the user queries the status of
1643 * the interface using ifconfig.
1645 **********************************************************************/
1647 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1649 struct ixl_vsi *vsi = ifp->if_softc;
1650 struct ixl_pf *pf = vsi->back;
1651 struct i40e_hw *hw = &pf->hw;
1653 INIT_DEBUGOUT("ixl_media_status: begin");
1656 hw->phy.get_link_info = TRUE;
1657 i40e_get_link_status(hw, &pf->link_up);
1658 ixl_update_link_status(pf);
1660 ifmr->ifm_status = IFM_AVALID;
1661 ifmr->ifm_active = IFM_ETHER;
1668 ifmr->ifm_status |= IFM_ACTIVE;
1670 /* Hardware always does full-duplex */
1671 ifmr->ifm_active |= IFM_FDX;
1673 switch (hw->phy.link_info.phy_type) {
1675 case I40E_PHY_TYPE_100BASE_TX:
1676 ifmr->ifm_active |= IFM_100_TX;
1679 case I40E_PHY_TYPE_1000BASE_T:
1680 ifmr->ifm_active |= IFM_1000_T;
1682 case I40E_PHY_TYPE_1000BASE_SX:
1683 ifmr->ifm_active |= IFM_1000_SX;
1685 case I40E_PHY_TYPE_1000BASE_LX:
1686 ifmr->ifm_active |= IFM_1000_LX;
1688 case I40E_PHY_TYPE_1000BASE_T_OPTICAL:
1689 ifmr->ifm_active |= IFM_OTHER;
1692 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1693 ifmr->ifm_active |= IFM_10G_TWINAX;
1695 case I40E_PHY_TYPE_10GBASE_SR:
1696 ifmr->ifm_active |= IFM_10G_SR;
1698 case I40E_PHY_TYPE_10GBASE_LR:
1699 ifmr->ifm_active |= IFM_10G_LR;
1701 case I40E_PHY_TYPE_10GBASE_T:
1702 ifmr->ifm_active |= IFM_10G_T;
1704 case I40E_PHY_TYPE_XAUI:
1705 case I40E_PHY_TYPE_XFI:
1706 case I40E_PHY_TYPE_10GBASE_AOC:
1707 ifmr->ifm_active |= IFM_OTHER;
1710 case I40E_PHY_TYPE_40GBASE_CR4:
1711 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1712 ifmr->ifm_active |= IFM_40G_CR4;
1714 case I40E_PHY_TYPE_40GBASE_SR4:
1715 ifmr->ifm_active |= IFM_40G_SR4;
1717 case I40E_PHY_TYPE_40GBASE_LR4:
1718 ifmr->ifm_active |= IFM_40G_LR4;
1720 case I40E_PHY_TYPE_XLAUI:
1721 ifmr->ifm_active |= IFM_OTHER;
1723 #ifndef IFM_ETH_XTYPE
1724 case I40E_PHY_TYPE_1000BASE_KX:
1725 ifmr->ifm_active |= IFM_1000_CX;
1727 case I40E_PHY_TYPE_SGMII:
1728 ifmr->ifm_active |= IFM_OTHER;
1730 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1731 case I40E_PHY_TYPE_10GBASE_CR1:
1732 ifmr->ifm_active |= IFM_10G_TWINAX;
1734 case I40E_PHY_TYPE_10GBASE_KX4:
1735 ifmr->ifm_active |= IFM_10G_CX4;
1737 case I40E_PHY_TYPE_10GBASE_KR:
1738 ifmr->ifm_active |= IFM_10G_SR;
1740 case I40E_PHY_TYPE_SFI:
1741 ifmr->ifm_active |= IFM_OTHER;
1743 case I40E_PHY_TYPE_40GBASE_KR4:
1744 case I40E_PHY_TYPE_XLPPI:
1745 case I40E_PHY_TYPE_40GBASE_AOC:
1746 ifmr->ifm_active |= IFM_40G_SR4;
1749 case I40E_PHY_TYPE_1000BASE_KX:
1750 ifmr->ifm_active |= IFM_1000_KX;
1752 case I40E_PHY_TYPE_SGMII:
1753 ifmr->ifm_active |= IFM_1000_SGMII;
1755 /* ERJ: What's the difference between these? */
1756 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1757 case I40E_PHY_TYPE_10GBASE_CR1:
1758 ifmr->ifm_active |= IFM_10G_CR1;
1760 case I40E_PHY_TYPE_10GBASE_KX4:
1761 ifmr->ifm_active |= IFM_10G_KX4;
1763 case I40E_PHY_TYPE_10GBASE_KR:
1764 ifmr->ifm_active |= IFM_10G_KR;
1766 case I40E_PHY_TYPE_SFI:
1767 ifmr->ifm_active |= IFM_10G_SFI;
1769 /* Our single 20G media type */
1770 case I40E_PHY_TYPE_20GBASE_KR2:
1771 ifmr->ifm_active |= IFM_20G_KR2;
1773 case I40E_PHY_TYPE_40GBASE_KR4:
1774 ifmr->ifm_active |= IFM_40G_KR4;
1776 case I40E_PHY_TYPE_XLPPI:
1777 case I40E_PHY_TYPE_40GBASE_AOC:
1778 ifmr->ifm_active |= IFM_40G_XLPPI;
1781 /* Unknown to driver */
1783 ifmr->ifm_active |= IFM_UNKNOWN;
1786 /* Report flow control status as well */
1787 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1788 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1789 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1790 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1798 * NOTE: Fortville does not support forcing media speeds. Instead,
1799 * use the set_advertise sysctl to set the speeds Fortville
1800 * will advertise or be allowed to operate at.
1803 ixl_media_change(struct ifnet * ifp)
1805 struct ixl_vsi *vsi = ifp->if_softc;
1806 struct ifmedia *ifm = &vsi->media;
1808 INIT_DEBUGOUT("ixl_media_change: begin");
1810 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1813 if_printf(ifp, "Media change is not supported.\n");
1821 ** ATR: Application Targetted Receive - creates a filter
1822 ** based on TX flow info that will keep the receive
1823 ** portion of the flow on the same queue. Based on the
1824 ** implementation this is only available for TCP connections
1827 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1829 struct ixl_vsi *vsi = que->vsi;
1830 struct tx_ring *txr = &que->txr;
1831 struct i40e_filter_program_desc *FDIR;
1835 /* check if ATR is enabled and sample rate */
1836 if ((!ixl_enable_fdir) || (!txr->atr_rate))
1839 ** We sample all TCP SYN/FIN packets,
1840 ** or at the selected sample rate
1843 if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1844 (txr->atr_count < txr->atr_rate))
1848 /* Get a descriptor to use */
1849 idx = txr->next_avail;
1850 FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1851 if (++idx == que->num_desc)
1854 txr->next_avail = idx;
1856 ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1857 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1859 ptype |= (etype == ETHERTYPE_IP) ?
1860 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1861 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1862 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1863 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1865 ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1867 dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1870 ** We use the TCP TH_FIN as a trigger to remove
1871 ** the filter, otherwise its an update.
1873 dtype |= (th->th_flags & TH_FIN) ?
1874 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1875 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1876 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1877 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1879 dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1880 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1882 dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1883 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1885 FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1886 FDIR->dtype_cmd_cntindex = htole32(dtype);
1893 ixl_set_promisc(struct ixl_vsi *vsi)
1895 struct ifnet *ifp = vsi->ifp;
1896 struct i40e_hw *hw = vsi->hw;
1898 bool uni = FALSE, multi = FALSE;
1900 if (ifp->if_flags & IFF_ALLMULTI)
1902 else { /* Need to count the multicast addresses */
1903 struct ifmultiaddr *ifma;
1904 if_maddr_rlock(ifp);
1905 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1906 if (ifma->ifma_addr->sa_family != AF_LINK)
1908 if (mcnt == MAX_MULTICAST_ADDR)
1912 if_maddr_runlock(ifp);
1915 if (mcnt >= MAX_MULTICAST_ADDR)
1917 if (ifp->if_flags & IFF_PROMISC)
1920 err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1921 vsi->seid, uni, NULL);
1922 err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1923 vsi->seid, multi, NULL);
1927 /*********************************************************************
1930 * Routines for multicast and vlan filter management.
1932 *********************************************************************/
1934 ixl_add_multi(struct ixl_vsi *vsi)
1936 struct ifmultiaddr *ifma;
1937 struct ifnet *ifp = vsi->ifp;
1938 struct i40e_hw *hw = vsi->hw;
1939 int mcnt = 0, flags;
1941 IOCTL_DEBUGOUT("ixl_add_multi: begin");
1943 if_maddr_rlock(ifp);
1945 ** First just get a count, to decide if we
1946 ** we simply use multicast promiscuous.
1948 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1949 if (ifma->ifma_addr->sa_family != AF_LINK)
1953 if_maddr_runlock(ifp);
1955 if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1956 /* delete existing MC filters */
1957 ixl_del_hw_filters(vsi, mcnt);
1958 i40e_aq_set_vsi_multicast_promiscuous(hw,
1959 vsi->seid, TRUE, NULL);
1964 if_maddr_rlock(ifp);
1965 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1966 if (ifma->ifma_addr->sa_family != AF_LINK)
1968 ixl_add_mc_filter(vsi,
1969 (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1972 if_maddr_runlock(ifp);
1974 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1975 ixl_add_hw_filters(vsi, flags, mcnt);
1978 IOCTL_DEBUGOUT("ixl_add_multi: end");
1983 ixl_del_multi(struct ixl_vsi *vsi)
1985 struct ifnet *ifp = vsi->ifp;
1986 struct ifmultiaddr *ifma;
1987 struct ixl_mac_filter *f;
1991 IOCTL_DEBUGOUT("ixl_del_multi: begin");
1993 /* Search for removed multicast addresses */
1994 if_maddr_rlock(ifp);
1995 SLIST_FOREACH(f, &vsi->ftl, next) {
1996 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1998 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1999 if (ifma->ifma_addr->sa_family != AF_LINK)
2001 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2002 if (cmp_etheraddr(f->macaddr, mc_addr)) {
2007 if (match == FALSE) {
2008 f->flags |= IXL_FILTER_DEL;
2013 if_maddr_runlock(ifp);
2016 ixl_del_hw_filters(vsi, mcnt);
2020 /*********************************************************************
2023 * This routine checks for link status,updates statistics,
2024 * and runs the watchdog check.
2026 * Only runs when the driver is configured UP and RUNNING.
2028 **********************************************************************/
2031 ixl_local_timer(void *arg)
2033 struct ixl_pf *pf = arg;
2034 struct i40e_hw *hw = &pf->hw;
2035 struct ixl_vsi *vsi = &pf->vsi;
2036 struct ixl_queue *que = vsi->queues;
2037 device_t dev = pf->dev;
2041 mtx_assert(&pf->pf_mtx, MA_OWNED);
2043 /* Fire off the adminq task */
2044 taskqueue_enqueue(pf->tq, &pf->adminq);
2047 ixl_update_stats_counters(pf);
2050 ** Check status of the queues
2052 mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
2053 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
2055 for (int i = 0; i < vsi->num_queues; i++, que++) {
2056 /* Any queues with outstanding work get a sw irq */
2058 wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
2060 ** Each time txeof runs without cleaning, but there
2061 ** are uncleaned descriptors it increments busy. If
2062 ** we get to 5 we declare it hung.
2064 if (que->busy == IXL_QUEUE_HUNG) {
2066 /* Mark the queue as inactive */
2067 vsi->active_queues &= ~((u64)1 << que->me);
2070 /* Check if we've come back from hung */
2071 if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
2072 vsi->active_queues |= ((u64)1 << que->me);
2074 if (que->busy >= IXL_MAX_TX_BUSY) {
2076 device_printf(dev,"Warning queue %d "
2077 "appears to be hung!\n", i);
2079 que->busy = IXL_QUEUE_HUNG;
2083 /* Only reinit if all queues show hung */
2084 if (hung == vsi->num_queues)
2087 callout_reset(&pf->timer, hz, ixl_local_timer, pf);
2091 device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
2092 ixl_init_locked(pf);
2096 ** Note: this routine updates the OS on the link state
2097 ** the real check of the hardware only happens with
2098 ** a link interrupt.
2101 ixl_update_link_status(struct ixl_pf *pf)
2103 struct ixl_vsi *vsi = &pf->vsi;
2104 struct i40e_hw *hw = &pf->hw;
2105 struct ifnet *ifp = vsi->ifp;
2106 device_t dev = pf->dev;
2109 if (vsi->link_active == FALSE) {
2110 pf->fc = hw->fc.current_mode;
2112 device_printf(dev,"Link is up %d Gbps %s,"
2113 " Flow Control: %s\n",
2115 I40E_LINK_SPEED_40GB)? 40:10),
2116 "Full Duplex", ixl_fc_string[pf->fc]);
2118 vsi->link_active = TRUE;
2120 ** Warn user if link speed on NPAR enabled
2121 ** partition is not at least 10GB
2123 if (hw->func_caps.npar_enable &&
2124 (hw->phy.link_info.link_speed ==
2125 I40E_LINK_SPEED_1GB ||
2126 hw->phy.link_info.link_speed ==
2127 I40E_LINK_SPEED_100MB))
2128 device_printf(dev, "The partition detected"
2129 "link speed that is less than 10Gbps\n");
2130 if_link_state_change(ifp, LINK_STATE_UP);
2132 } else { /* Link down */
2133 if (vsi->link_active == TRUE) {
2135 device_printf(dev, "Link is Down\n");
2136 if_link_state_change(ifp, LINK_STATE_DOWN);
2137 vsi->link_active = FALSE;
2145 ixl_stop(struct ixl_pf *pf)
2148 ixl_stop_locked(pf);
2151 ixl_teardown_queue_msix(&pf->vsi);
2152 ixl_free_queue_tqs(&pf->vsi);
2155 /*********************************************************************
2157 * This routine disables all traffic on the adapter by issuing a
2158 * global reset on the MAC and deallocates TX/RX buffers.
2160 **********************************************************************/
2163 ixl_stop_locked(struct ixl_pf *pf)
2165 struct ixl_vsi *vsi = &pf->vsi;
2166 struct ifnet *ifp = vsi->ifp;
2168 INIT_DEBUGOUT("ixl_stop: begin\n");
2170 IXL_PF_LOCK_ASSERT(pf);
2172 /* Stop the local timer */
2173 callout_stop(&pf->timer);
2175 ixl_disable_rings_intr(vsi);
2176 ixl_disable_rings(vsi);
2178 /* Tell the stack that the interface is no longer active */
2179 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING);
2183 /*********************************************************************
2185 * Setup MSIX Interrupt resources and handlers for the VSI
2187 **********************************************************************/
2189 ixl_assign_vsi_legacy(struct ixl_pf *pf)
2191 device_t dev = pf->dev;
2192 struct ixl_vsi *vsi = &pf->vsi;
2193 struct ixl_queue *que = vsi->queues;
2198 pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2199 &rid, RF_SHAREABLE | RF_ACTIVE);
2200 if (pf->res == NULL) {
2201 device_printf(dev, "Unable to allocate"
2202 " bus resource: vsi legacy/msi interrupt\n");
2206 /* Set the handler function */
2207 error = bus_setup_intr(dev, pf->res,
2208 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2209 ixl_intr, pf, &pf->tag);
2212 device_printf(dev, "Failed to register legacy/msi handler\n");
2215 bus_describe_intr(dev, pf->res, pf->tag, "irq0");
2216 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2217 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2218 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2219 taskqueue_thread_enqueue, &que->tq);
2220 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2221 device_get_nameunit(dev));
2222 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2224 pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2225 taskqueue_thread_enqueue, &pf->tq);
2226 taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2227 device_get_nameunit(dev));
2233 ixl_setup_adminq_tq(struct ixl_pf *pf)
2235 device_t dev = pf->dev;
2238 /* Tasklet for Admin Queue interrupts */
2239 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2242 TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2244 /* Create and start Admin Queue taskqueue */
2245 pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT,
2246 taskqueue_thread_enqueue, &pf->tq);
2248 device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n");
2251 error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq",
2252 device_get_nameunit(dev));
2254 device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n",
2256 taskqueue_free(pf->tq);
2263 ixl_setup_queue_tqs(struct ixl_vsi *vsi)
2265 struct ixl_queue *que = vsi->queues;
2266 device_t dev = vsi->dev;
2272 /* Create queue tasks and start queue taskqueues */
2273 for (int i = 0; i < vsi->num_queues; i++, que++) {
2274 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2275 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2276 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2277 taskqueue_thread_enqueue, &que->tq);
2279 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2280 CPU_SETOF(cpu_id, &cpu_mask);
2281 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2282 &cpu_mask, "%s (bucket %d)",
2283 device_get_nameunit(dev), cpu_id);
2285 taskqueue_start_threads(&que->tq, 1, PI_NET,
2286 "%s (que %d)", device_get_nameunit(dev), que->me);
2294 ixl_free_adminq_tq(struct ixl_pf *pf)
2297 taskqueue_free(pf->tq);
2303 ixl_free_queue_tqs(struct ixl_vsi *vsi)
2305 struct ixl_queue *que = vsi->queues;
2307 for (int i = 0; i < vsi->num_queues; i++, que++) {
2309 taskqueue_free(que->tq);
2316 ixl_setup_adminq_msix(struct ixl_pf *pf)
2318 device_t dev = pf->dev;
2321 /* Admin IRQ rid is 1, vector is 0 */
2323 /* Get interrupt resource from bus */
2324 pf->res = bus_alloc_resource_any(dev,
2325 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2327 device_printf(dev, "bus_alloc_resource_any() for Admin Queue"
2328 " interrupt failed [rid=%d]\n", rid);
2331 /* Then associate interrupt with handler */
2332 error = bus_setup_intr(dev, pf->res,
2333 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2334 ixl_msix_adminq, pf, &pf->tag);
2337 device_printf(dev, "bus_setup_intr() for Admin Queue"
2338 " interrupt handler failed, error %d\n", error);
2341 error = bus_describe_intr(dev, pf->res, pf->tag, "aq");
2343 /* Probably non-fatal? */
2344 device_printf(dev, "bus_describe_intr() for Admin Queue"
2345 " interrupt name failed, error %d\n", error);
2353 * Allocate interrupt resources from bus and associate an interrupt handler
2354 * to those for the VSI's queues.
2357 ixl_setup_queue_msix(struct ixl_vsi *vsi)
2359 device_t dev = vsi->dev;
2360 struct ixl_queue *que = vsi->queues;
2361 struct tx_ring *txr;
2362 int error, rid, vector = 1;
2364 /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */
2365 for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2369 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2370 RF_SHAREABLE | RF_ACTIVE);
2372 device_printf(dev, "bus_alloc_resource_any() for"
2373 " Queue %d interrupt failed [rid=%d]\n",
2377 /* Set the handler function */
2378 error = bus_setup_intr(dev, que->res,
2379 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2380 ixl_msix_que, que, &que->tag);
2382 device_printf(dev, "bus_setup_intr() for Queue %d"
2383 " interrupt handler failed, error %d\n",
2385 // TODO: Check for error from this?
2386 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2389 error = bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2391 device_printf(dev, "bus_describe_intr() for Queue %d"
2392 " interrupt name failed, error %d\n",
2395 /* Bind the vector to a CPU */
2397 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2399 error = bus_bind_intr(dev, que->res, cpu_id);
2401 device_printf(dev, "bus_bind_intr() for Queue %d"
2402 " to CPU %d failed, error %d\n",
2403 que->me, cpu_id, error);
2413 * Allocate MSI/X vectors
2416 ixl_init_msix(struct ixl_pf *pf)
2418 device_t dev = pf->dev;
2419 int rid, want, vectors, queues, available;
2421 /* Override by tuneable */
2422 if (ixl_enable_msix == 0)
2426 ** When used in a virtualized environment
2427 ** PCI BUSMASTER capability may not be set
2428 ** so explicity set it here and rewrite
2429 ** the ENABLE in the MSIX control register
2430 ** at this point to cause the host to
2431 ** successfully initialize us.
2436 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2437 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2438 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2439 pci_find_cap(dev, PCIY_MSIX, &rid);
2440 rid += PCIR_MSIX_CTRL;
2441 msix_ctrl = pci_read_config(dev, rid, 2);
2442 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2443 pci_write_config(dev, rid, msix_ctrl, 2);
2446 /* First try MSI/X */
2447 rid = PCIR_BAR(IXL_BAR);
2448 pf->msix_mem = bus_alloc_resource_any(dev,
2449 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2450 if (!pf->msix_mem) {
2451 /* May not be enabled */
2452 device_printf(pf->dev,
2453 "Unable to map MSIX table\n");
2457 available = pci_msix_count(dev);
2458 if (available == 0) { /* system has msix disabled */
2459 bus_release_resource(dev, SYS_RES_MEMORY,
2461 pf->msix_mem = NULL;
2465 /* Figure out a reasonable auto config value */
2466 queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2468 /* Override with tunable value if tunable is less than autoconfig count */
2469 if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2470 queues = ixl_max_queues;
2471 else if ((ixl_max_queues != 0) && (ixl_max_queues > queues))
2472 device_printf(dev, "ixl_max_queues > # of cpus, using "
2473 "autoconfig amount...\n");
2474 /* Or limit maximum auto-configured queues to 8 */
2475 else if ((ixl_max_queues == 0) && (queues > 8))
2479 /* If we're doing RSS, clamp at the number of RSS buckets */
2480 if (queues > rss_getnumbuckets())
2481 queues = rss_getnumbuckets();
2485 ** Want one vector (RX/TX pair) per queue
2486 ** plus an additional for the admin queue.
2489 if (want <= available) /* Have enough */
2492 device_printf(pf->dev,
2493 "MSIX Configuration Problem, "
2494 "%d vectors available but %d wanted!\n",
2496 return (0); /* Will go to Legacy setup */
2499 if (pci_alloc_msix(dev, &vectors) == 0) {
2500 device_printf(pf->dev,
2501 "Using MSIX interrupts with %d vectors\n", vectors);
2503 pf->vsi.num_queues = queues;
2506 * If we're doing RSS, the number of queues needs to
2507 * match the number of RSS buckets that are configured.
2509 * + If there's more queues than RSS buckets, we'll end
2510 * up with queues that get no traffic.
2512 * + If there's more RSS buckets than queues, we'll end
2513 * up having multiple RSS buckets map to the same queue,
2514 * so there'll be some contention.
2516 if (queues != rss_getnumbuckets()) {
2518 "%s: queues (%d) != RSS buckets (%d)"
2519 "; performance will be impacted.\n",
2520 __func__, queues, rss_getnumbuckets());
2526 vectors = pci_msi_count(dev);
2527 pf->vsi.num_queues = 1;
2529 ixl_enable_msix = 0;
2530 if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2531 device_printf(pf->dev, "Using an MSI interrupt\n");
2534 device_printf(pf->dev, "Using a Legacy interrupt\n");
2540 * Configure admin queue/misc interrupt cause registers in hardware.
2543 ixl_configure_intr0_msix(struct ixl_pf *pf)
2545 struct i40e_hw *hw = &pf->hw;
2548 /* First set up the adminq - vector 0 */
2549 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
2550 rd32(hw, I40E_PFINT_ICR0); /* read to clear */
2552 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2553 I40E_PFINT_ICR0_ENA_GRST_MASK |
2554 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
2555 I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2556 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2557 I40E_PFINT_ICR0_ENA_VFLR_MASK |
2558 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2559 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2562 * 0x7FF is the end of the queue list.
2563 * This means we won't use MSI-X vector 0 for a queue interrupt
2566 wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2567 /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */
2568 wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E);
2570 wr32(hw, I40E_PFINT_DYN_CTL0,
2571 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2572 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2574 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2578 * Configure queue interrupt cause registers in hardware.
2581 ixl_configure_queue_intr_msix(struct ixl_pf *pf)
2583 struct i40e_hw *hw = &pf->hw;
2584 struct ixl_vsi *vsi = &pf->vsi;
2588 for (int i = 0; i < vsi->num_queues; i++, vector++) {
2589 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
2590 /* First queue type is RX / 0 */
2591 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2593 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2594 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2595 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2596 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2597 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2598 wr32(hw, I40E_QINT_RQCTL(i), reg);
2600 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2601 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2602 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2603 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2604 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2605 wr32(hw, I40E_QINT_TQCTL(i), reg);
2610 * Configure for MSI single vector operation
2613 ixl_configure_legacy(struct ixl_pf *pf)
2615 struct i40e_hw *hw = &pf->hw;
2618 wr32(hw, I40E_PFINT_ITR0(0), 0);
2619 wr32(hw, I40E_PFINT_ITR0(1), 0);
2621 /* Setup "other" causes */
2622 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2623 | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2624 | I40E_PFINT_ICR0_ENA_GRST_MASK
2625 | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2626 | I40E_PFINT_ICR0_ENA_GPIO_MASK
2627 | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2628 | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2629 | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2630 | I40E_PFINT_ICR0_ENA_VFLR_MASK
2631 | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2633 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2635 /* SW_ITR_IDX = 0, but don't change INTENA */
2636 wr32(hw, I40E_PFINT_DYN_CTL0,
2637 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2638 I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2639 /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2640 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2642 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2643 wr32(hw, I40E_PFINT_LNKLST0, 0);
2645 /* Associate the queue pair to the vector and enable the q int */
2646 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2647 | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2648 | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2649 wr32(hw, I40E_QINT_RQCTL(0), reg);
2651 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2652 | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2653 | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2654 wr32(hw, I40E_QINT_TQCTL(0), reg);
2659 * Get initial ITR values from tunable values.
2662 ixl_configure_itr(struct ixl_pf *pf)
2664 struct i40e_hw *hw = &pf->hw;
2665 struct ixl_vsi *vsi = &pf->vsi;
2666 struct ixl_queue *que = vsi->queues;
2668 vsi->rx_itr_setting = ixl_rx_itr;
2669 vsi->tx_itr_setting = ixl_tx_itr;
2671 for (int i = 0; i < vsi->num_queues; i++, que++) {
2672 struct tx_ring *txr = &que->txr;
2673 struct rx_ring *rxr = &que->rxr;
2675 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2676 vsi->rx_itr_setting);
2677 rxr->itr = vsi->rx_itr_setting;
2678 rxr->latency = IXL_AVE_LATENCY;
2680 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2681 vsi->tx_itr_setting);
2682 txr->itr = vsi->tx_itr_setting;
2683 txr->latency = IXL_AVE_LATENCY;
2689 ixl_allocate_pci_resources(struct ixl_pf *pf)
2692 device_t dev = pf->dev;
2695 pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2698 if (!(pf->pci_mem)) {
2699 device_printf(dev, "Unable to allocate bus resource: PCI memory\n");
2703 pf->osdep.mem_bus_space_tag =
2704 rman_get_bustag(pf->pci_mem);
2705 pf->osdep.mem_bus_space_handle =
2706 rman_get_bushandle(pf->pci_mem);
2707 pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2708 pf->osdep.flush_reg = I40E_GLGEN_STAT;
2709 pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2711 pf->hw.back = &pf->osdep;
2714 ** Now setup MSI or MSI/X, should
2715 ** return us the number of supported
2716 ** vectors. (Will be 1 for MSI)
2718 pf->msix = ixl_init_msix(pf);
2723 * Teardown and release the admin queue/misc vector
2727 ixl_teardown_adminq_msix(struct ixl_pf *pf)
2729 device_t dev = pf->dev;
2732 if (pf->admvec) /* we are doing MSIX */
2733 rid = pf->admvec + 1;
2735 (pf->msix != 0) ? (rid = 1):(rid = 0);
2737 // TODO: Check for errors from bus_teardown_intr
2738 // TODO: Check for errors from bus_release_resource
2739 if (pf->tag != NULL) {
2740 bus_teardown_intr(dev, pf->res, pf->tag);
2743 if (pf->res != NULL) {
2744 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2752 ixl_teardown_queue_msix(struct ixl_vsi *vsi)
2754 struct ixl_queue *que = vsi->queues;
2755 device_t dev = vsi->dev;
2758 /* We may get here before stations are setup */
2759 if ((!ixl_enable_msix) || (que == NULL))
2762 /* Release all MSIX queue resources */
2763 for (int i = 0; i < vsi->num_queues; i++, que++) {
2764 rid = que->msix + 1;
2765 if (que->tag != NULL) {
2766 error = bus_teardown_intr(dev, que->res, que->tag);
2768 device_printf(dev, "bus_teardown_intr() for"
2769 " Queue %d interrupt failed\n",
2775 if (que->res != NULL) {
2776 error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2778 device_printf(dev, "bus_release_resource() for"
2779 " Queue %d interrupt failed [rid=%d]\n",
2791 ixl_free_pci_resources(struct ixl_pf *pf)
2793 device_t dev = pf->dev;
2796 ixl_teardown_queue_msix(&pf->vsi);
2797 ixl_teardown_adminq_msix(pf);
2800 pci_release_msi(dev);
2802 memrid = PCIR_BAR(IXL_BAR);
2804 if (pf->msix_mem != NULL)
2805 bus_release_resource(dev, SYS_RES_MEMORY,
2806 memrid, pf->msix_mem);
2808 if (pf->pci_mem != NULL)
2809 bus_release_resource(dev, SYS_RES_MEMORY,
2810 PCIR_BAR(0), pf->pci_mem);
2816 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2818 /* Display supported media types */
2819 if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2820 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2822 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2823 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2824 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2825 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2826 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2827 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2829 if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2830 phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2831 phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2832 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2834 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2835 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2836 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2837 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2838 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2839 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2841 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2842 phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2843 phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2844 phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2845 phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2846 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2847 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2848 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2849 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2850 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2852 #ifndef IFM_ETH_XTYPE
2853 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2854 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2856 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2857 phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2858 phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2859 phy_type & (1 << I40E_PHY_TYPE_SFI))
2860 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2861 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2862 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2863 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2864 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2866 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2867 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2868 if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2869 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2871 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2872 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2874 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2875 || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2876 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2877 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2878 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2879 if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2880 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2881 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2882 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2883 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2884 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2886 if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2887 ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2889 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2890 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2891 if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2892 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2896 /*********************************************************************
2898 * Setup networking device structure and register an interface.
2900 **********************************************************************/
2902 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2905 struct i40e_hw *hw = vsi->hw;
2906 struct ixl_queue *que = vsi->queues;
2907 struct i40e_aq_get_phy_abilities_resp abilities;
2908 enum i40e_status_code aq_error = 0;
2910 INIT_DEBUGOUT("ixl_setup_interface: begin");
2912 ifp = vsi->ifp = if_alloc(IFT_ETHER);
2914 device_printf(dev, "can not allocate ifnet structure\n");
2917 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2918 ifp->if_mtu = ETHERMTU;
2919 ifp->if_baudrate = IF_Gbps(40);
2920 ifp->if_init = ixl_init;
2921 ifp->if_softc = vsi;
2922 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2923 ifp->if_ioctl = ixl_ioctl;
2925 #if __FreeBSD_version >= 1100036
2926 if_setgetcounterfn(ifp, ixl_get_counter);
2929 ifp->if_transmit = ixl_mq_start;
2931 ifp->if_qflush = ixl_qflush;
2933 ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2935 vsi->max_frame_size =
2936 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2937 + ETHER_VLAN_ENCAP_LEN;
2940 * Tell the upper layer(s) we support long frames.
2942 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2944 ifp->if_capabilities |= IFCAP_HWCSUM;
2945 ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2946 ifp->if_capabilities |= IFCAP_TSO;
2947 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2948 ifp->if_capabilities |= IFCAP_LRO;
2950 /* VLAN capabilties */
2951 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2954 | IFCAP_VLAN_HWCSUM;
2955 ifp->if_capenable = ifp->if_capabilities;
2958 ** Don't turn this on by default, if vlans are
2959 ** created on another pseudo device (eg. lagg)
2960 ** then vlan events are not passed thru, breaking
2961 ** operation, but with HW FILTER off it works. If
2962 ** using vlans directly on the ixl driver you can
2963 ** enable this and get full hardware tag filtering.
2965 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2968 * Specify the media types supported by this adapter and register
2969 * callbacks to update media and link information
2971 ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2974 aq_error = i40e_aq_get_phy_capabilities(hw,
2975 FALSE, TRUE, &abilities, NULL);
2976 /* May need delay to detect fiber correctly */
2977 if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2978 i40e_msec_delay(200);
2979 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2980 TRUE, &abilities, NULL);
2983 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2984 device_printf(dev, "Unknown PHY type detected!\n");
2987 "Error getting supported media types, err %d,"
2988 " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2992 ixl_add_ifmedia(vsi, abilities.phy_type);
2994 /* Use autoselect media by default */
2995 ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2996 ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2998 ether_ifattach(ifp, hw->mac.addr);
3004 ** Run when the Admin Queue gets a link state change interrupt.
3007 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
3009 struct i40e_hw *hw = &pf->hw;
3010 device_t dev = pf->dev;
3011 struct i40e_aqc_get_link_status *status =
3012 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
3014 /* Request link status from adapter */
3015 hw->phy.get_link_info = TRUE;
3016 i40e_get_link_status(hw, &pf->link_up);
3018 /* Print out message if an unqualified module is found */
3019 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
3020 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
3021 (!(status->link_info & I40E_AQ_LINK_UP)))
3022 device_printf(dev, "Link failed because "
3023 "an unqualified module was detected!\n");
3025 /* Update OS link info */
3026 ixl_update_link_status(pf);
3029 /*********************************************************************
3031 * Get Firmware Switch configuration
3032 * - this will need to be more robust when more complex
3033 * switch configurations are enabled.
3035 **********************************************************************/
3037 ixl_switch_config(struct ixl_pf *pf)
3039 struct i40e_hw *hw = &pf->hw;
3040 struct ixl_vsi *vsi = &pf->vsi;
3041 device_t dev = vsi->dev;
3042 struct i40e_aqc_get_switch_config_resp *sw_config;
3043 u8 aq_buf[I40E_AQ_LARGE_BUF];
3047 memset(&aq_buf, 0, sizeof(aq_buf));
3048 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
3049 ret = i40e_aq_get_switch_config(hw, sw_config,
3050 sizeof(aq_buf), &next, NULL);
3052 device_printf(dev, "aq_get_switch_config() failed, error %d,"
3053 " aq_error %d\n", ret, pf->hw.aq.asq_last_status);
3058 "Switch config: header reported: %d in structure, %d total\n",
3059 sw_config->header.num_reported, sw_config->header.num_total);
3060 for (int i = 0; i < sw_config->header.num_reported; i++) {
3062 "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
3063 sw_config->element[i].element_type,
3064 sw_config->element[i].seid,
3065 sw_config->element[i].uplink_seid,
3066 sw_config->element[i].downlink_seid);
3069 /* Simplified due to a single VSI at the moment */
3070 vsi->uplink_seid = sw_config->element[0].uplink_seid;
3071 vsi->downlink_seid = sw_config->element[0].downlink_seid;
3072 vsi->seid = sw_config->element[0].seid;
3076 /*********************************************************************
3078 * Initialize the VSI: this handles contexts, which means things
3079 * like the number of descriptors, buffer size,
3080 * plus we init the rings thru this function.
3082 **********************************************************************/
3084 ixl_initialize_vsi(struct ixl_vsi *vsi)
3086 struct ixl_pf *pf = vsi->back;
3087 struct ixl_queue *que = vsi->queues;
3088 device_t dev = vsi->dev;
3089 struct i40e_hw *hw = vsi->hw;
3090 struct i40e_vsi_context ctxt;
3093 memset(&ctxt, 0, sizeof(ctxt));
3094 ctxt.seid = vsi->seid;
3095 if (pf->veb_seid != 0)
3096 ctxt.uplink_seid = pf->veb_seid;
3097 ctxt.pf_num = hw->pf_id;
3098 err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
3100 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d"
3101 " aq_error %d\n", err, hw->aq.asq_last_status);
3105 device_printf(dev, "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
3106 "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
3107 "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
3108 ctxt.uplink_seid, ctxt.vsi_number,
3109 ctxt.vsis_allocated, ctxt.vsis_unallocated,
3110 ctxt.flags, ctxt.pf_num, ctxt.vf_num,
3111 ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
3114 ** Set the queue and traffic class bits
3115 ** - when multiple traffic classes are supported
3116 ** this will need to be more robust.
3118 ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
3119 ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
3120 /* In contig mode, que_mapping[0] is first queue index used by this VSI */
3121 ctxt.info.queue_mapping[0] = 0;
3123 * This VSI will only use traffic class 0; start traffic class 0's
3124 * queue allocation at queue 0, and assign it 64 (2^6) queues (though
3125 * the driver may not use all of them).
3127 ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
3128 & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
3129 ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
3130 & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
3132 /* Set VLAN receive stripping mode */
3133 ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
3134 ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
3135 if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
3136 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3138 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3140 /* Keep copy of VSI info in VSI for statistic counters */
3141 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
3143 /* Reset VSI statistics */
3144 ixl_vsi_reset_stats(vsi);
3145 vsi->hw_filters_add = 0;
3146 vsi->hw_filters_del = 0;
3148 ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
3150 err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3152 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d, aq_error %d\n",
3153 err, hw->aq.asq_last_status);
3157 for (int i = 0; i < vsi->num_queues; i++, que++) {
3158 struct tx_ring *txr = &que->txr;
3159 struct rx_ring *rxr = &que->rxr;
3160 struct i40e_hmc_obj_txq tctx;
3161 struct i40e_hmc_obj_rxq rctx;
3165 /* Setup the HMC TX Context */
3166 size = que->num_desc * sizeof(struct i40e_tx_desc);
3167 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
3168 tctx.new_context = 1;
3169 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
3170 tctx.qlen = que->num_desc;
3172 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
3173 /* Enable HEAD writeback */
3174 tctx.head_wb_ena = 1;
3175 tctx.head_wb_addr = txr->dma.pa +
3176 (que->num_desc * sizeof(struct i40e_tx_desc));
3177 tctx.rdylist_act = 0;
3178 err = i40e_clear_lan_tx_queue_context(hw, i);
3180 device_printf(dev, "Unable to clear TX context\n");
3183 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
3185 device_printf(dev, "Unable to set TX context\n");
3188 /* Associate the ring with this PF */
3189 txctl = I40E_QTX_CTL_PF_QUEUE;
3190 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3191 I40E_QTX_CTL_PF_INDX_MASK);
3192 wr32(hw, I40E_QTX_CTL(i), txctl);
3195 /* Do ring (re)init */
3196 ixl_init_tx_ring(que);
3198 /* Next setup the HMC RX Context */
3199 if (vsi->max_frame_size <= MCLBYTES)
3200 rxr->mbuf_sz = MCLBYTES;
3202 rxr->mbuf_sz = MJUMPAGESIZE;
3204 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
3206 /* Set up an RX context for the HMC */
3207 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
3208 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
3209 /* ignore header split for now */
3210 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
3211 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
3212 vsi->max_frame_size : max_rxmax;
3214 rctx.dsize = 1; /* do 32byte descriptors */
3215 rctx.hsplit_0 = 0; /* no HDR split initially */
3216 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
3217 rctx.qlen = que->num_desc;
3218 rctx.tphrdesc_ena = 1;
3219 rctx.tphwdesc_ena = 1;
3220 rctx.tphdata_ena = 0;
3221 rctx.tphhead_ena = 0;
3222 rctx.lrxqthresh = 2;
3229 err = i40e_clear_lan_rx_queue_context(hw, i);
3232 "Unable to clear RX context %d\n", i);
3235 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
3237 device_printf(dev, "Unable to set RX context %d\n", i);
3240 err = ixl_init_rx_ring(que);
3242 device_printf(dev, "Fail in init_rx_ring %d\n", i);
3246 /* preserve queue */
3247 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
3248 struct netmap_adapter *na = NA(vsi->ifp);
3249 struct netmap_kring *kring = &na->rx_rings[i];
3250 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
3251 wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
3253 #endif /* DEV_NETMAP */
3254 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
3260 /*********************************************************************
3262 * Free all VSI structs.
3264 **********************************************************************/
3266 ixl_free_vsi(struct ixl_vsi *vsi)
3268 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3269 struct ixl_queue *que = vsi->queues;
3271 /* Free station queues */
3275 for (int i = 0; i < vsi->num_queues; i++, que++) {
3276 struct tx_ring *txr = &que->txr;
3277 struct rx_ring *rxr = &que->rxr;
3279 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
3282 ixl_free_que_tx(que);
3284 i40e_free_dma_mem(&pf->hw, &txr->dma);
3286 IXL_TX_LOCK_DESTROY(txr);
3288 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
3291 ixl_free_que_rx(que);
3293 i40e_free_dma_mem(&pf->hw, &rxr->dma);
3295 IXL_RX_LOCK_DESTROY(rxr);
3298 free(vsi->queues, M_DEVBUF);
3301 /* Free VSI filter list */
3302 ixl_free_mac_filters(vsi);
3306 ixl_free_mac_filters(struct ixl_vsi *vsi)
3308 struct ixl_mac_filter *f;
3310 while (!SLIST_EMPTY(&vsi->ftl)) {
3311 f = SLIST_FIRST(&vsi->ftl);
3312 SLIST_REMOVE_HEAD(&vsi->ftl, next);
3318 /*********************************************************************
3320 * Allocate memory for the VSI (virtual station interface) and their
3321 * associated queues, rings and the descriptors associated with each,
3322 * called only once at attach.
3324 **********************************************************************/
3326 ixl_setup_stations(struct ixl_pf *pf)
3328 device_t dev = pf->dev;
3329 struct ixl_vsi *vsi;
3330 struct ixl_queue *que;
3331 struct tx_ring *txr;
3332 struct rx_ring *rxr;
3334 int error = I40E_SUCCESS;
3337 vsi->back = (void *)pf;
3343 /* Get memory for the station queues */
3345 (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
3346 vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3347 device_printf(dev, "Unable to allocate queue memory\n");
3352 for (int i = 0; i < vsi->num_queues; i++) {
3353 que = &vsi->queues[i];
3354 que->num_desc = ixl_ringsz;
3357 /* mark the queue as active */
3358 vsi->active_queues |= (u64)1 << que->me;
3361 txr->tail = I40E_QTX_TAIL(que->me);
3363 /* Initialize the TX lock */
3364 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3365 device_get_nameunit(dev), que->me);
3366 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
3367 /* Create the TX descriptor ring */
3368 tsize = roundup2((que->num_desc *
3369 sizeof(struct i40e_tx_desc)) +
3370 sizeof(u32), DBA_ALIGN);
3371 if (i40e_allocate_dma_mem(&pf->hw,
3372 &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
3374 "Unable to allocate TX Descriptor memory\n");
3378 txr->base = (struct i40e_tx_desc *)txr->dma.va;
3379 bzero((void *)txr->base, tsize);
3380 /* Now allocate transmit soft structs for the ring */
3381 if (ixl_allocate_tx_data(que)) {
3383 "Critical Failure setting up TX structures\n");
3387 /* Allocate a buf ring */
3388 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3389 M_NOWAIT, &txr->mtx);
3390 if (txr->br == NULL) {
3392 "Critical Failure setting up TX buf ring\n");
3398 * Next the RX queues...
3400 rsize = roundup2(que->num_desc *
3401 sizeof(union i40e_rx_desc), DBA_ALIGN);
3404 rxr->tail = I40E_QRX_TAIL(que->me);
3406 /* Initialize the RX side lock */
3407 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3408 device_get_nameunit(dev), que->me);
3409 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3411 if (i40e_allocate_dma_mem(&pf->hw,
3412 &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3414 "Unable to allocate RX Descriptor memory\n");
3418 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3419 bzero((void *)rxr->base, rsize);
3421 /* Allocate receive soft structs for the ring*/
3422 if (ixl_allocate_rx_data(que)) {
3424 "Critical Failure setting up receive structs\n");
3433 for (int i = 0; i < vsi->num_queues; i++) {
3434 que = &vsi->queues[i];
3438 i40e_free_dma_mem(&pf->hw, &rxr->dma);
3440 i40e_free_dma_mem(&pf->hw, &txr->dma);
3448 ** Provide a update to the queue RX
3449 ** interrupt moderation value.
3452 ixl_set_queue_rx_itr(struct ixl_queue *que)
3454 struct ixl_vsi *vsi = que->vsi;
3455 struct i40e_hw *hw = vsi->hw;
3456 struct rx_ring *rxr = &que->rxr;
3461 /* Idle, do nothing */
3462 if (rxr->bytes == 0)
3465 if (ixl_dynamic_rx_itr) {
3466 rx_bytes = rxr->bytes/rxr->itr;
3469 /* Adjust latency range */
3470 switch (rxr->latency) {
3471 case IXL_LOW_LATENCY:
3472 if (rx_bytes > 10) {
3473 rx_latency = IXL_AVE_LATENCY;
3474 rx_itr = IXL_ITR_20K;
3477 case IXL_AVE_LATENCY:
3478 if (rx_bytes > 20) {
3479 rx_latency = IXL_BULK_LATENCY;
3480 rx_itr = IXL_ITR_8K;
3481 } else if (rx_bytes <= 10) {
3482 rx_latency = IXL_LOW_LATENCY;
3483 rx_itr = IXL_ITR_100K;
3486 case IXL_BULK_LATENCY:
3487 if (rx_bytes <= 20) {
3488 rx_latency = IXL_AVE_LATENCY;
3489 rx_itr = IXL_ITR_20K;
3494 rxr->latency = rx_latency;
3496 if (rx_itr != rxr->itr) {
3497 /* do an exponential smoothing */
3498 rx_itr = (10 * rx_itr * rxr->itr) /
3499 ((9 * rx_itr) + rxr->itr);
3500 rxr->itr = rx_itr & IXL_MAX_ITR;
3501 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3502 que->me), rxr->itr);
3504 } else { /* We may have have toggled to non-dynamic */
3505 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3506 vsi->rx_itr_setting = ixl_rx_itr;
3507 /* Update the hardware if needed */
3508 if (rxr->itr != vsi->rx_itr_setting) {
3509 rxr->itr = vsi->rx_itr_setting;
3510 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3511 que->me), rxr->itr);
3521 ** Provide a update to the queue TX
3522 ** interrupt moderation value.
3525 ixl_set_queue_tx_itr(struct ixl_queue *que)
3527 struct ixl_vsi *vsi = que->vsi;
3528 struct i40e_hw *hw = vsi->hw;
3529 struct tx_ring *txr = &que->txr;
3535 /* Idle, do nothing */
3536 if (txr->bytes == 0)
3539 if (ixl_dynamic_tx_itr) {
3540 tx_bytes = txr->bytes/txr->itr;
3543 switch (txr->latency) {
3544 case IXL_LOW_LATENCY:
3545 if (tx_bytes > 10) {
3546 tx_latency = IXL_AVE_LATENCY;
3547 tx_itr = IXL_ITR_20K;
3550 case IXL_AVE_LATENCY:
3551 if (tx_bytes > 20) {
3552 tx_latency = IXL_BULK_LATENCY;
3553 tx_itr = IXL_ITR_8K;
3554 } else if (tx_bytes <= 10) {
3555 tx_latency = IXL_LOW_LATENCY;
3556 tx_itr = IXL_ITR_100K;
3559 case IXL_BULK_LATENCY:
3560 if (tx_bytes <= 20) {
3561 tx_latency = IXL_AVE_LATENCY;
3562 tx_itr = IXL_ITR_20K;
3567 txr->latency = tx_latency;
3569 if (tx_itr != txr->itr) {
3570 /* do an exponential smoothing */
3571 tx_itr = (10 * tx_itr * txr->itr) /
3572 ((9 * tx_itr) + txr->itr);
3573 txr->itr = tx_itr & IXL_MAX_ITR;
3574 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3575 que->me), txr->itr);
3578 } else { /* We may have have toggled to non-dynamic */
3579 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3580 vsi->tx_itr_setting = ixl_tx_itr;
3581 /* Update the hardware if needed */
3582 if (txr->itr != vsi->tx_itr_setting) {
3583 txr->itr = vsi->tx_itr_setting;
3584 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3585 que->me), txr->itr);
3593 #define QUEUE_NAME_LEN 32
3596 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3597 struct sysctl_ctx_list *ctx, const char *sysctl_name)
3599 struct sysctl_oid *tree;
3600 struct sysctl_oid_list *child;
3601 struct sysctl_oid_list *vsi_list;
3603 tree = device_get_sysctl_tree(pf->dev);
3604 child = SYSCTL_CHILDREN(tree);
3605 vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3606 CTLFLAG_RD, NULL, "VSI Number");
3607 vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3609 ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3614 * ixl_sysctl_qtx_tail_handler
3615 * Retrieves I40E_QTX_TAIL value from hardware
3619 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)
3621 struct ixl_queue *que;
3625 que = ((struct ixl_queue *)oidp->oid_arg1);
3628 val = rd32(que->vsi->hw, que->txr.tail);
3629 error = sysctl_handle_int(oidp, &val, 0, req);
3630 if (error || !req->newptr)
3636 * ixl_sysctl_qrx_tail_handler
3637 * Retrieves I40E_QRX_TAIL value from hardware
3641 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)
3643 struct ixl_queue *que;
3647 que = ((struct ixl_queue *)oidp->oid_arg1);
3650 val = rd32(que->vsi->hw, que->rxr.tail);
3651 error = sysctl_handle_int(oidp, &val, 0, req);
3652 if (error || !req->newptr)
3659 ixl_add_hw_stats(struct ixl_pf *pf)
3661 device_t dev = pf->dev;
3662 struct ixl_vsi *vsi = &pf->vsi;
3663 struct ixl_queue *queues = vsi->queues;
3664 struct i40e_hw_port_stats *pf_stats = &pf->stats;
3666 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3667 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3668 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3669 struct sysctl_oid_list *vsi_list;
3671 struct sysctl_oid *queue_node;
3672 struct sysctl_oid_list *queue_list;
3674 struct tx_ring *txr;
3675 struct rx_ring *rxr;
3676 char queue_namebuf[QUEUE_NAME_LEN];
3678 /* Driver statistics */
3679 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3680 CTLFLAG_RD, &pf->watchdog_events,
3681 "Watchdog timeouts");
3682 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3683 CTLFLAG_RD, &pf->admin_irq,
3684 "Admin Queue IRQ Handled");
3686 ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3687 vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3689 /* Queue statistics */
3690 for (int q = 0; q < vsi->num_queues; q++) {
3691 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3692 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3693 OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3694 queue_list = SYSCTL_CHILDREN(queue_node);
3696 txr = &(queues[q].txr);
3697 rxr = &(queues[q].rxr);
3699 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3700 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3701 "m_defrag() failed");
3702 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3703 CTLFLAG_RD, &(queues[q].irqs),
3704 "irqs on this queue");
3705 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3706 CTLFLAG_RD, &(queues[q].tso),
3708 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3709 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3710 "Driver tx dma failure in xmit");
3711 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3712 CTLFLAG_RD, &(txr->no_desc),
3713 "Queue No Descriptor Available");
3714 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3715 CTLFLAG_RD, &(txr->total_packets),
3716 "Queue Packets Transmitted");
3717 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3718 CTLFLAG_RD, &(txr->tx_bytes),
3719 "Queue Bytes Transmitted");
3720 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3721 CTLFLAG_RD, &(rxr->rx_packets),
3722 "Queue Packets Received");
3723 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3724 CTLFLAG_RD, &(rxr->rx_bytes),
3725 "Queue Bytes Received");
3726 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err",
3727 CTLFLAG_RD, &(rxr->desc_errs),
3728 "Queue Rx Descriptor Errors");
3729 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr",
3730 CTLFLAG_RD, &(rxr->itr), 0,
3731 "Queue Rx ITR Interval");
3732 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr",
3733 CTLFLAG_RD, &(txr->itr), 0,
3734 "Queue Tx ITR Interval");
3735 // Not actual latency; just a calculated value to put in a register
3736 // TODO: Put in better descriptions here
3737 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_latency",
3738 CTLFLAG_RD, &(rxr->latency), 0,
3739 "Queue Rx ITRL Average Interval");
3740 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_latency",
3741 CTLFLAG_RD, &(txr->latency), 0,
3742 "Queue Tx ITRL Average Interval");
3745 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done",
3746 CTLFLAG_RD, &(rxr->not_done),
3747 "Queue Rx Descriptors not Done");
3748 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh",
3749 CTLFLAG_RD, &(rxr->next_refresh), 0,
3750 "Queue Rx Descriptors not Done");
3751 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check",
3752 CTLFLAG_RD, &(rxr->next_check), 0,
3753 "Queue Rx Descriptors not Done");
3754 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail",
3755 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3756 sizeof(struct ixl_queue),
3757 ixl_sysctl_qtx_tail_handler, "IU",
3758 "Queue Transmit Descriptor Tail");
3759 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail",
3760 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3761 sizeof(struct ixl_queue),
3762 ixl_sysctl_qrx_tail_handler, "IU",
3763 "Queue Receive Descriptor Tail");
3768 ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3772 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3773 struct sysctl_oid_list *child,
3774 struct i40e_eth_stats *eth_stats)
3776 struct ixl_sysctl_info ctls[] =
3778 {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3779 {ð_stats->rx_unicast, "ucast_pkts_rcvd",
3780 "Unicast Packets Received"},
3781 {ð_stats->rx_multicast, "mcast_pkts_rcvd",
3782 "Multicast Packets Received"},
3783 {ð_stats->rx_broadcast, "bcast_pkts_rcvd",
3784 "Broadcast Packets Received"},
3785 {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3786 {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3787 {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3788 {ð_stats->tx_multicast, "mcast_pkts_txd",
3789 "Multicast Packets Transmitted"},
3790 {ð_stats->tx_broadcast, "bcast_pkts_txd",
3791 "Broadcast Packets Transmitted"},
3796 struct ixl_sysctl_info *entry = ctls;
3797 while (entry->stat != NULL)
3799 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3800 CTLFLAG_RD, entry->stat,
3801 entry->description);
3807 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3808 struct sysctl_oid_list *child,
3809 struct i40e_hw_port_stats *stats)
3811 struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3812 CTLFLAG_RD, NULL, "Mac Statistics");
3813 struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3815 struct i40e_eth_stats *eth_stats = &stats->eth;
3816 ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3818 struct ixl_sysctl_info ctls[] =
3820 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3821 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3822 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3823 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3824 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3825 /* Packet Reception Stats */
3826 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3827 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3828 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3829 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3830 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3831 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3832 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3833 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3834 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3835 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3836 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3837 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3838 /* Packet Transmission Stats */
3839 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3840 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3841 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3842 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3843 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3844 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3845 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3847 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3848 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3849 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3850 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3855 struct ixl_sysctl_info *entry = ctls;
3856 while (entry->stat != NULL)
3858 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3859 CTLFLAG_RD, entry->stat,
3860 entry->description);
3867 ** ixl_config_rss - setup RSS
3868 ** - note this is done for the single vsi
3871 ixl_config_rss(struct ixl_vsi *vsi)
3873 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3874 struct i40e_hw *hw = vsi->hw;
3876 u64 set_hena = 0, hena;
3879 u32 rss_hash_config;
3880 u32 rss_seed[IXL_KEYSZ];
3882 u32 rss_seed[IXL_KEYSZ] = {0x41b01687,
3883 0x183cfd8c, 0xce880440, 0x580cbc3c,
3884 0x35897377, 0x328b25e1, 0x4fa98922,
3885 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3889 /* Fetch the configured RSS key */
3890 rss_getkey((uint8_t *) &rss_seed);
3893 /* Fill out hash function seed */
3894 for (i = 0; i < IXL_KEYSZ; i++)
3895 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3897 /* Enable PCTYPES for RSS: */
3899 rss_hash_config = rss_gethashconfig();
3900 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3901 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3902 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3903 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3904 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3905 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3906 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3907 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3908 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3909 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3910 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3911 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3912 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3913 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3916 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3917 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3918 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3919 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3920 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3921 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3922 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3923 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3924 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3925 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3926 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3928 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
3929 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
3931 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
3932 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3934 /* Populate the LUT with max no. of queues in round robin fashion */
3935 for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3936 if (j == vsi->num_queues)
3940 * Fetch the RSS bucket id for the given indirection entry.
3941 * Cap it at the number of configured buckets (which is
3944 que_id = rss_get_indirection_to_bucket(i);
3945 que_id = que_id % vsi->num_queues;
3949 /* lut = 4-byte sliding window of 4 lut entries */
3950 lut = (lut << 8) | (que_id &
3951 ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3952 /* On i = 3, we have 4 entries in lut; write to the register */
3954 wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3961 ** This routine is run via an vlan config EVENT,
3962 ** it enables us to use the HW Filter table since
3963 ** we can get the vlan id. This just creates the
3964 ** entry in the soft version of the VFTA, init will
3965 ** repopulate the real table.
3968 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3970 struct ixl_vsi *vsi = ifp->if_softc;
3971 struct i40e_hw *hw = vsi->hw;
3972 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3974 if (ifp->if_softc != arg) /* Not our event */
3977 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
3982 ixl_add_filter(vsi, hw->mac.addr, vtag);
3987 ** This routine is run via an vlan
3988 ** unconfig EVENT, remove our entry
3989 ** in the soft vfta.
3992 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3994 struct ixl_vsi *vsi = ifp->if_softc;
3995 struct i40e_hw *hw = vsi->hw;
3996 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3998 if (ifp->if_softc != arg)
4001 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4006 ixl_del_filter(vsi, hw->mac.addr, vtag);
4011 ** This routine updates vlan filters, called by init
4012 ** it scans the filter table and then updates the hw
4013 ** after a soft reset.
4016 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
4018 struct ixl_mac_filter *f;
4021 if (vsi->num_vlans == 0)
4024 ** Scan the filter list for vlan entries,
4025 ** mark them for addition and then call
4026 ** for the AQ update.
4028 SLIST_FOREACH(f, &vsi->ftl, next) {
4029 if (f->flags & IXL_FILTER_VLAN) {
4037 printf("setup vlan: no filters found!\n");
4040 flags = IXL_FILTER_VLAN;
4041 flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4042 ixl_add_hw_filters(vsi, flags, cnt);
4047 ** Initialize filter list and add filters that the hardware
4048 ** needs to know about.
4050 ** Requires VSI's filter list & seid to be set before calling.
4053 ixl_init_filters(struct ixl_vsi *vsi)
4055 /* Add broadcast address */
4056 ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
4059 * Prevent Tx flow control frames from being sent out by
4060 * non-firmware transmitters.
4062 i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid);
4066 ** This routine adds mulicast filters
4069 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
4071 struct ixl_mac_filter *f;
4073 /* Does one already exist */
4074 f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4078 f = ixl_get_filter(vsi);
4080 printf("WARNING: no filter available!!\n");
4083 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4084 f->vlan = IXL_VLAN_ANY;
4085 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
4092 ixl_reconfigure_filters(struct ixl_vsi *vsi)
4095 ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
4099 ** This routine adds macvlan filters
4102 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4104 struct ixl_mac_filter *f, *tmp;
4108 DEBUGOUT("ixl_add_filter: begin");
4113 /* Does one already exist */
4114 f = ixl_find_filter(vsi, macaddr, vlan);
4118 ** Is this the first vlan being registered, if so we
4119 ** need to remove the ANY filter that indicates we are
4120 ** not in a vlan, and replace that with a 0 filter.
4122 if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
4123 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4125 ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
4126 ixl_add_filter(vsi, macaddr, 0);
4130 f = ixl_get_filter(vsi);
4132 device_printf(dev, "WARNING: no filter available!!\n");
4135 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4137 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4138 if (f->vlan != IXL_VLAN_ANY)
4139 f->flags |= IXL_FILTER_VLAN;
4143 ixl_add_hw_filters(vsi, f->flags, 1);
4148 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4150 struct ixl_mac_filter *f;
4152 f = ixl_find_filter(vsi, macaddr, vlan);
4156 f->flags |= IXL_FILTER_DEL;
4157 ixl_del_hw_filters(vsi, 1);
4160 /* Check if this is the last vlan removal */
4161 if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
4162 /* Switch back to a non-vlan filter */
4163 ixl_del_filter(vsi, macaddr, 0);
4164 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
4170 ** Find the filter with both matching mac addr and vlan id
4172 static struct ixl_mac_filter *
4173 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4175 struct ixl_mac_filter *f;
4178 SLIST_FOREACH(f, &vsi->ftl, next) {
4179 if (!cmp_etheraddr(f->macaddr, macaddr))
4181 if (f->vlan == vlan) {
4193 ** This routine takes additions to the vsi filter
4194 ** table and creates an Admin Queue call to create
4195 ** the filters in the hardware.
4198 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
4200 struct i40e_aqc_add_macvlan_element_data *a, *b;
4201 struct ixl_mac_filter *f;
4210 IXL_PF_LOCK_ASSERT(pf);
4212 a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
4213 M_DEVBUF, M_NOWAIT | M_ZERO);
4215 device_printf(dev, "add_hw_filters failed to get memory\n");
4220 ** Scan the filter list, each time we find one
4221 ** we add it to the admin queue array and turn off
4224 SLIST_FOREACH(f, &vsi->ftl, next) {
4225 if (f->flags == flags) {
4226 b = &a[j]; // a pox on fvl long names :)
4227 bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
4228 if (f->vlan == IXL_VLAN_ANY) {
4230 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
4232 b->vlan_tag = f->vlan;
4235 b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
4236 f->flags &= ~IXL_FILTER_ADD;
4243 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
4245 device_printf(dev, "aq_add_macvlan err %d, "
4246 "aq_error %d\n", err, hw->aq.asq_last_status);
4248 vsi->hw_filters_add += j;
4255 ** This routine takes removals in the vsi filter
4256 ** table and creates an Admin Queue call to delete
4257 ** the filters in the hardware.
4260 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
4262 struct i40e_aqc_remove_macvlan_element_data *d, *e;
4266 struct ixl_mac_filter *f, *f_temp;
4269 DEBUGOUT("ixl_del_hw_filters: begin\n");
4275 d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
4276 M_DEVBUF, M_NOWAIT | M_ZERO);
4278 printf("del hw filter failed to get memory\n");
4282 SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
4283 if (f->flags & IXL_FILTER_DEL) {
4284 e = &d[j]; // a pox on fvl long names :)
4285 bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
4286 e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
4287 e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
4288 /* delete entry from vsi list */
4289 SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
4297 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
4298 /* NOTE: returns ENOENT every time but seems to work fine,
4299 so we'll ignore that specific error. */
4300 // TODO: Does this still occur on current firmwares?
4301 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
4303 for (int i = 0; i < j; i++)
4304 sc += (!d[i].error_code);
4305 vsi->hw_filters_del += sc;
4307 "Failed to remove %d/%d filters, aq error %d\n",
4308 j - sc, j, hw->aq.asq_last_status);
4310 vsi->hw_filters_del += j;
4314 DEBUGOUT("ixl_del_hw_filters: end\n");
4319 ixl_enable_rings(struct ixl_vsi *vsi)
4321 struct ixl_pf *pf = vsi->back;
4322 struct i40e_hw *hw = &pf->hw;
4327 for (int i = 0; i < vsi->num_queues; i++) {
4328 index = vsi->first_queue + i;
4329 i40e_pre_tx_queue_cfg(hw, index, TRUE);
4331 reg = rd32(hw, I40E_QTX_ENA(index));
4332 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
4333 I40E_QTX_ENA_QENA_STAT_MASK;
4334 wr32(hw, I40E_QTX_ENA(index), reg);
4335 /* Verify the enable took */
4336 for (int j = 0; j < 10; j++) {
4337 reg = rd32(hw, I40E_QTX_ENA(index));
4338 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4340 i40e_msec_delay(10);
4342 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
4343 device_printf(pf->dev, "TX queue %d disabled!\n",
4348 reg = rd32(hw, I40E_QRX_ENA(index));
4349 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
4350 I40E_QRX_ENA_QENA_STAT_MASK;
4351 wr32(hw, I40E_QRX_ENA(index), reg);
4352 /* Verify the enable took */
4353 for (int j = 0; j < 10; j++) {
4354 reg = rd32(hw, I40E_QRX_ENA(index));
4355 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
4357 i40e_msec_delay(10);
4359 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
4360 device_printf(pf->dev, "RX queue %d disabled!\n",
4370 ixl_disable_rings(struct ixl_vsi *vsi)
4372 struct ixl_pf *pf = vsi->back;
4373 struct i40e_hw *hw = &pf->hw;
4378 for (int i = 0; i < vsi->num_queues; i++) {
4379 index = vsi->first_queue + i;
4381 i40e_pre_tx_queue_cfg(hw, index, FALSE);
4382 i40e_usec_delay(500);
4384 reg = rd32(hw, I40E_QTX_ENA(index));
4385 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4386 wr32(hw, I40E_QTX_ENA(index), reg);
4387 /* Verify the disable took */
4388 for (int j = 0; j < 10; j++) {
4389 reg = rd32(hw, I40E_QTX_ENA(index));
4390 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4392 i40e_msec_delay(10);
4394 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
4395 device_printf(pf->dev, "TX queue %d still enabled!\n",
4400 reg = rd32(hw, I40E_QRX_ENA(index));
4401 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4402 wr32(hw, I40E_QRX_ENA(index), reg);
4403 /* Verify the disable took */
4404 for (int j = 0; j < 10; j++) {
4405 reg = rd32(hw, I40E_QRX_ENA(index));
4406 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
4408 i40e_msec_delay(10);
4410 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
4411 device_printf(pf->dev, "RX queue %d still enabled!\n",
4421 * ixl_handle_mdd_event
4423 * Called from interrupt handler to identify possibly malicious vfs
4424 * (But also detects events from the PF, as well)
4427 ixl_handle_mdd_event(struct ixl_pf *pf)
4429 struct i40e_hw *hw = &pf->hw;
4430 device_t dev = pf->dev;
4431 bool mdd_detected = false;
4432 bool pf_mdd_detected = false;
4435 /* find what triggered the MDD event */
4436 reg = rd32(hw, I40E_GL_MDET_TX);
4437 if (reg & I40E_GL_MDET_TX_VALID_MASK) {
4438 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
4439 I40E_GL_MDET_TX_PF_NUM_SHIFT;
4440 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
4441 I40E_GL_MDET_TX_EVENT_SHIFT;
4442 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
4443 I40E_GL_MDET_TX_QUEUE_SHIFT;
4445 "Malicious Driver Detection event 0x%02x"
4446 " on TX queue %d pf number 0x%02x\n",
4447 event, queue, pf_num);
4448 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
4449 mdd_detected = true;
4451 reg = rd32(hw, I40E_GL_MDET_RX);
4452 if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4453 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
4454 I40E_GL_MDET_RX_FUNCTION_SHIFT;
4455 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
4456 I40E_GL_MDET_RX_EVENT_SHIFT;
4457 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
4458 I40E_GL_MDET_RX_QUEUE_SHIFT;
4460 "Malicious Driver Detection event 0x%02x"
4461 " on RX queue %d of function 0x%02x\n",
4462 event, queue, func);
4463 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4464 mdd_detected = true;
4468 reg = rd32(hw, I40E_PF_MDET_TX);
4469 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4470 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4472 "MDD TX event is for this function 0x%08x",
4474 pf_mdd_detected = true;
4476 reg = rd32(hw, I40E_PF_MDET_RX);
4477 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4478 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4480 "MDD RX event is for this function 0x%08x",
4482 pf_mdd_detected = true;
4486 /* re-enable mdd interrupt cause */
4487 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4488 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4489 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4494 ixl_enable_intr(struct ixl_vsi *vsi)
4496 struct i40e_hw *hw = vsi->hw;
4497 struct ixl_queue *que = vsi->queues;
4499 if (ixl_enable_msix) {
4500 for (int i = 0; i < vsi->num_queues; i++, que++)
4501 ixl_enable_queue(hw, que->me);
4503 ixl_enable_legacy(hw);
4507 ixl_disable_rings_intr(struct ixl_vsi *vsi)
4509 struct i40e_hw *hw = vsi->hw;
4510 struct ixl_queue *que = vsi->queues;
4512 for (int i = 0; i < vsi->num_queues; i++, que++)
4513 ixl_disable_queue(hw, que->me);
4517 ixl_disable_intr(struct ixl_vsi *vsi)
4519 struct i40e_hw *hw = vsi->hw;
4521 if (ixl_enable_msix)
4522 ixl_disable_adminq(hw);
4524 ixl_disable_legacy(hw);
4528 ixl_enable_adminq(struct i40e_hw *hw)
4532 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4533 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4534 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4535 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4540 ixl_disable_adminq(struct i40e_hw *hw)
4544 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4545 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4550 ixl_enable_queue(struct i40e_hw *hw, int id)
4554 reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4555 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4556 (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4557 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4561 ixl_disable_queue(struct i40e_hw *hw, int id)
4565 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4566 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4570 ixl_enable_legacy(struct i40e_hw *hw)
4573 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4574 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4575 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4576 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4580 ixl_disable_legacy(struct i40e_hw *hw)
4584 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4585 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4589 ixl_update_stats_counters(struct ixl_pf *pf)
4591 struct i40e_hw *hw = &pf->hw;
4592 struct ixl_vsi *vsi = &pf->vsi;
4595 struct i40e_hw_port_stats *nsd = &pf->stats;
4596 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4598 /* Update hw stats */
4599 ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4600 pf->stat_offsets_loaded,
4601 &osd->crc_errors, &nsd->crc_errors);
4602 ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4603 pf->stat_offsets_loaded,
4604 &osd->illegal_bytes, &nsd->illegal_bytes);
4605 ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4606 I40E_GLPRT_GORCL(hw->port),
4607 pf->stat_offsets_loaded,
4608 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4609 ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4610 I40E_GLPRT_GOTCL(hw->port),
4611 pf->stat_offsets_loaded,
4612 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4613 ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4614 pf->stat_offsets_loaded,
4615 &osd->eth.rx_discards,
4616 &nsd->eth.rx_discards);
4617 ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4618 I40E_GLPRT_UPRCL(hw->port),
4619 pf->stat_offsets_loaded,
4620 &osd->eth.rx_unicast,
4621 &nsd->eth.rx_unicast);
4622 ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4623 I40E_GLPRT_UPTCL(hw->port),
4624 pf->stat_offsets_loaded,
4625 &osd->eth.tx_unicast,
4626 &nsd->eth.tx_unicast);
4627 ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4628 I40E_GLPRT_MPRCL(hw->port),
4629 pf->stat_offsets_loaded,
4630 &osd->eth.rx_multicast,
4631 &nsd->eth.rx_multicast);
4632 ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4633 I40E_GLPRT_MPTCL(hw->port),
4634 pf->stat_offsets_loaded,
4635 &osd->eth.tx_multicast,
4636 &nsd->eth.tx_multicast);
4637 ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4638 I40E_GLPRT_BPRCL(hw->port),
4639 pf->stat_offsets_loaded,
4640 &osd->eth.rx_broadcast,
4641 &nsd->eth.rx_broadcast);
4642 ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4643 I40E_GLPRT_BPTCL(hw->port),
4644 pf->stat_offsets_loaded,
4645 &osd->eth.tx_broadcast,
4646 &nsd->eth.tx_broadcast);
4648 ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4649 pf->stat_offsets_loaded,
4650 &osd->tx_dropped_link_down,
4651 &nsd->tx_dropped_link_down);
4652 ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4653 pf->stat_offsets_loaded,
4654 &osd->mac_local_faults,
4655 &nsd->mac_local_faults);
4656 ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4657 pf->stat_offsets_loaded,
4658 &osd->mac_remote_faults,
4659 &nsd->mac_remote_faults);
4660 ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4661 pf->stat_offsets_loaded,
4662 &osd->rx_length_errors,
4663 &nsd->rx_length_errors);
4665 /* Flow control (LFC) stats */
4666 ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4667 pf->stat_offsets_loaded,
4668 &osd->link_xon_rx, &nsd->link_xon_rx);
4669 ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4670 pf->stat_offsets_loaded,
4671 &osd->link_xon_tx, &nsd->link_xon_tx);
4672 ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4673 pf->stat_offsets_loaded,
4674 &osd->link_xoff_rx, &nsd->link_xoff_rx);
4675 ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4676 pf->stat_offsets_loaded,
4677 &osd->link_xoff_tx, &nsd->link_xoff_tx);
4679 /* Packet size stats rx */
4680 ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4681 I40E_GLPRT_PRC64L(hw->port),
4682 pf->stat_offsets_loaded,
4683 &osd->rx_size_64, &nsd->rx_size_64);
4684 ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4685 I40E_GLPRT_PRC127L(hw->port),
4686 pf->stat_offsets_loaded,
4687 &osd->rx_size_127, &nsd->rx_size_127);
4688 ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4689 I40E_GLPRT_PRC255L(hw->port),
4690 pf->stat_offsets_loaded,
4691 &osd->rx_size_255, &nsd->rx_size_255);
4692 ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4693 I40E_GLPRT_PRC511L(hw->port),
4694 pf->stat_offsets_loaded,
4695 &osd->rx_size_511, &nsd->rx_size_511);
4696 ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4697 I40E_GLPRT_PRC1023L(hw->port),
4698 pf->stat_offsets_loaded,
4699 &osd->rx_size_1023, &nsd->rx_size_1023);
4700 ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4701 I40E_GLPRT_PRC1522L(hw->port),
4702 pf->stat_offsets_loaded,
4703 &osd->rx_size_1522, &nsd->rx_size_1522);
4704 ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4705 I40E_GLPRT_PRC9522L(hw->port),
4706 pf->stat_offsets_loaded,
4707 &osd->rx_size_big, &nsd->rx_size_big);
4709 /* Packet size stats tx */
4710 ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4711 I40E_GLPRT_PTC64L(hw->port),
4712 pf->stat_offsets_loaded,
4713 &osd->tx_size_64, &nsd->tx_size_64);
4714 ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4715 I40E_GLPRT_PTC127L(hw->port),
4716 pf->stat_offsets_loaded,
4717 &osd->tx_size_127, &nsd->tx_size_127);
4718 ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4719 I40E_GLPRT_PTC255L(hw->port),
4720 pf->stat_offsets_loaded,
4721 &osd->tx_size_255, &nsd->tx_size_255);
4722 ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4723 I40E_GLPRT_PTC511L(hw->port),
4724 pf->stat_offsets_loaded,
4725 &osd->tx_size_511, &nsd->tx_size_511);
4726 ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4727 I40E_GLPRT_PTC1023L(hw->port),
4728 pf->stat_offsets_loaded,
4729 &osd->tx_size_1023, &nsd->tx_size_1023);
4730 ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4731 I40E_GLPRT_PTC1522L(hw->port),
4732 pf->stat_offsets_loaded,
4733 &osd->tx_size_1522, &nsd->tx_size_1522);
4734 ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4735 I40E_GLPRT_PTC9522L(hw->port),
4736 pf->stat_offsets_loaded,
4737 &osd->tx_size_big, &nsd->tx_size_big);
4739 ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4740 pf->stat_offsets_loaded,
4741 &osd->rx_undersize, &nsd->rx_undersize);
4742 ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4743 pf->stat_offsets_loaded,
4744 &osd->rx_fragments, &nsd->rx_fragments);
4745 ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4746 pf->stat_offsets_loaded,
4747 &osd->rx_oversize, &nsd->rx_oversize);
4748 ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4749 pf->stat_offsets_loaded,
4750 &osd->rx_jabber, &nsd->rx_jabber);
4751 pf->stat_offsets_loaded = true;
4754 /* Update vsi stats */
4755 ixl_update_vsi_stats(vsi);
4757 for (int i = 0; i < pf->num_vfs; i++) {
4759 if (vf->vf_flags & VF_FLAG_ENABLED)
4760 ixl_update_eth_stats(&pf->vfs[i].vsi);
4765 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf)
4767 struct i40e_hw *hw = &pf->hw;
4768 struct ixl_vsi *vsi = &pf->vsi;
4769 device_t dev = pf->dev;
4773 is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING);
4778 error = i40e_shutdown_lan_hmc(hw);
4781 "Shutdown LAN HMC failed with code %d\n", error);
4782 ixl_disable_adminq(hw);
4783 ixl_teardown_adminq_msix(pf);
4784 error = i40e_shutdown_adminq(hw);
4787 "Shutdown Admin queue failed with code %d\n", error);
4790 error = i40e_init_adminq(hw);
4791 if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
4792 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
4795 error = ixl_setup_adminq_msix(pf);
4797 device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
4800 ixl_configure_intr0_msix(pf);
4801 ixl_enable_adminq(hw);
4802 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4803 hw->func_caps.num_rx_qp, 0, 0);
4805 device_printf(dev, "init_lan_hmc failed: %d\n", error);
4807 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4809 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
4818 ixl_handle_empr_reset(struct ixl_pf *pf)
4820 struct i40e_hw *hw = &pf->hw;
4821 device_t dev = pf->dev;
4825 /* Typically finishes within 3-4 seconds */
4826 while (count++ < 100) {
4827 reg = rd32(hw, I40E_GLGEN_RSTAT)
4828 & I40E_GLGEN_RSTAT_DEVSTATE_MASK;
4830 i40e_msec_delay(100);
4836 device_printf(dev, "EMPR reset wait count: %d\n", count);
4839 device_printf(dev, "Rebuilding driver state...\n");
4840 ixl_rebuild_hw_structs_after_reset(pf);
4841 device_printf(dev, "Rebuilding driver state done.\n");
4843 atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
4847 ** Tasklet handler for MSIX Adminq interrupts
4848 ** - do outside interrupt since it might sleep
4851 ixl_do_adminq(void *context, int pending)
4853 struct ixl_pf *pf = context;
4854 struct i40e_hw *hw = &pf->hw;
4855 struct i40e_arq_event_info event;
4857 device_t dev = pf->dev;
4861 if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
4862 /* Flag cleared at end of this function */
4863 ixl_handle_empr_reset(pf);
4867 /* Admin Queue handling */
4868 event.buf_len = IXL_AQ_BUF_SZ;
4869 event.msg_buf = malloc(event.buf_len,
4870 M_DEVBUF, M_NOWAIT | M_ZERO);
4871 if (!event.msg_buf) {
4872 device_printf(dev, "%s: Unable to allocate memory for Admin"
4873 " Queue event!\n", __func__);
4878 /* clean and process any events */
4880 ret = i40e_clean_arq_element(hw, &event, &result);
4883 opcode = LE16_TO_CPU(event.desc.opcode);
4885 device_printf(dev, "%s: Admin Queue event: %#06x\n", __func__,
4889 case i40e_aqc_opc_get_link_status:
4890 ixl_link_event(pf, &event);
4892 case i40e_aqc_opc_send_msg_to_pf:
4894 ixl_handle_vf_msg(pf, &event);
4897 case i40e_aqc_opc_event_lan_overflow:
4902 } while (result && (loop++ < IXL_ADM_LIMIT));
4904 free(event.msg_buf, M_DEVBUF);
4907 * If there are still messages to process, reschedule ourselves.
4908 * Otherwise, re-enable our interrupt and go to sleep.
4911 taskqueue_enqueue(pf->tq, &pf->adminq);
4913 ixl_enable_adminq(hw);
4919 * Update VSI-specific ethernet statistics counters.
4922 ixl_update_eth_stats(struct ixl_vsi *vsi)
4924 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4925 struct i40e_hw *hw = &pf->hw;
4926 struct i40e_eth_stats *es;
4927 struct i40e_eth_stats *oes;
4928 struct i40e_hw_port_stats *nsd;
4929 u16 stat_idx = vsi->info.stat_counter_idx;
4931 es = &vsi->eth_stats;
4932 oes = &vsi->eth_stats_offsets;
4935 /* Gather up the stats that the hw collects */
4936 ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4937 vsi->stat_offsets_loaded,
4938 &oes->tx_errors, &es->tx_errors);
4939 ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4940 vsi->stat_offsets_loaded,
4941 &oes->rx_discards, &es->rx_discards);
4943 ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4944 I40E_GLV_GORCL(stat_idx),
4945 vsi->stat_offsets_loaded,
4946 &oes->rx_bytes, &es->rx_bytes);
4947 ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4948 I40E_GLV_UPRCL(stat_idx),
4949 vsi->stat_offsets_loaded,
4950 &oes->rx_unicast, &es->rx_unicast);
4951 ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4952 I40E_GLV_MPRCL(stat_idx),
4953 vsi->stat_offsets_loaded,
4954 &oes->rx_multicast, &es->rx_multicast);
4955 ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4956 I40E_GLV_BPRCL(stat_idx),
4957 vsi->stat_offsets_loaded,
4958 &oes->rx_broadcast, &es->rx_broadcast);
4960 ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4961 I40E_GLV_GOTCL(stat_idx),
4962 vsi->stat_offsets_loaded,
4963 &oes->tx_bytes, &es->tx_bytes);
4964 ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4965 I40E_GLV_UPTCL(stat_idx),
4966 vsi->stat_offsets_loaded,
4967 &oes->tx_unicast, &es->tx_unicast);
4968 ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4969 I40E_GLV_MPTCL(stat_idx),
4970 vsi->stat_offsets_loaded,
4971 &oes->tx_multicast, &es->tx_multicast);
4972 ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4973 I40E_GLV_BPTCL(stat_idx),
4974 vsi->stat_offsets_loaded,
4975 &oes->tx_broadcast, &es->tx_broadcast);
4976 vsi->stat_offsets_loaded = true;
4980 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4984 struct i40e_eth_stats *es;
4987 struct i40e_hw_port_stats *nsd;
4991 es = &vsi->eth_stats;
4994 ixl_update_eth_stats(vsi);
4996 tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4997 for (int i = 0; i < vsi->num_queues; i++)
4998 tx_discards += vsi->queues[i].txr.br->br_drops;
5000 /* Update ifnet stats */
5001 IXL_SET_IPACKETS(vsi, es->rx_unicast +
5004 IXL_SET_OPACKETS(vsi, es->tx_unicast +
5007 IXL_SET_IBYTES(vsi, es->rx_bytes);
5008 IXL_SET_OBYTES(vsi, es->tx_bytes);
5009 IXL_SET_IMCASTS(vsi, es->rx_multicast);
5010 IXL_SET_OMCASTS(vsi, es->tx_multicast);
5012 IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
5013 nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
5015 IXL_SET_OERRORS(vsi, es->tx_errors);
5016 IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
5017 IXL_SET_OQDROPS(vsi, tx_discards);
5018 IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
5019 IXL_SET_COLLISIONS(vsi, 0);
5023 * Reset all of the stats for the given pf
5025 void ixl_pf_reset_stats(struct ixl_pf *pf)
5027 bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
5028 bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
5029 pf->stat_offsets_loaded = false;
5033 * Resets all stats of the given vsi
5035 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
5037 bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
5038 bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
5039 vsi->stat_offsets_loaded = false;
5043 * Read and update a 48 bit stat from the hw
5045 * Since the device stats are not reset at PFReset, they likely will not
5046 * be zeroed when the driver starts. We'll save the first values read
5047 * and use them as offsets to be subtracted from the raw values in order
5048 * to report stats that count from zero.
5051 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
5052 bool offset_loaded, u64 *offset, u64 *stat)
5056 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
5057 new_data = rd64(hw, loreg);
5060 * Use two rd32's instead of one rd64; FreeBSD versions before
5061 * 10 don't support 8 byte bus reads/writes.
5063 new_data = rd32(hw, loreg);
5064 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
5069 if (new_data >= *offset)
5070 *stat = new_data - *offset;
5072 *stat = (new_data + ((u64)1 << 48)) - *offset;
5073 *stat &= 0xFFFFFFFFFFFFULL;
5077 * Read and update a 32 bit stat from the hw
5080 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
5081 bool offset_loaded, u64 *offset, u64 *stat)
5085 new_data = rd32(hw, reg);
5088 if (new_data >= *offset)
5089 *stat = (u32)(new_data - *offset);
5091 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
5095 ixl_add_device_sysctls(struct ixl_pf *pf)
5097 device_t dev = pf->dev;
5099 /* Set up sysctls */
5100 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5101 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5102 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
5103 pf, 0, ixl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC);
5105 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5106 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5107 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
5108 pf, 0, ixl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE);
5110 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5111 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5112 OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
5113 pf, 0, ixl_current_speed, "A", "Current Port Speed");
5115 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5116 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5117 OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
5118 pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
5121 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5122 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5123 OID_AUTO, "rx_itr", CTLFLAG_RW,
5124 &ixl_rx_itr, 0, "RX ITR");
5126 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5127 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5128 OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
5129 &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
5131 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5132 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5133 OID_AUTO, "tx_itr", CTLFLAG_RW,
5134 &ixl_tx_itr, 0, "TX ITR");
5136 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5137 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5138 OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
5139 &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
5142 #ifdef IXL_DEBUG_SYSCTL
5143 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5144 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5145 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
5146 ixl_debug_info, "I", "Debug Information");
5148 /* Shared-code debug message level */
5149 SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5150 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5151 OID_AUTO, "debug_mask", CTLFLAG_RW,
5152 &pf->hw.debug_mask, 0, "Debug Message Level");
5154 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5155 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5156 OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
5157 pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS);
5159 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5160 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5161 OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
5162 pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
5164 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5165 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5166 OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
5167 pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
5169 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5170 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5171 OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
5172 pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
5174 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5175 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5176 OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
5177 pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
5180 SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5181 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5182 OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
5183 0, "PF/VF Virtual Channel debug level");
5189 ** Set flow control using sysctl:
5196 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
5199 * TODO: ensure tx CRC by hardware should be enabled
5200 * if tx flow control is enabled.
5201 * ^ N/A for 40G ports
5203 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5204 struct i40e_hw *hw = &pf->hw;
5205 device_t dev = pf->dev;
5206 int requested_fc, error = 0;
5207 enum i40e_status_code aq_error = 0;
5211 requested_fc = pf->fc;
5212 error = sysctl_handle_int(oidp, &requested_fc, 0, req);
5213 if ((error) || (req->newptr == NULL))
5215 if (requested_fc < 0 || requested_fc > 3) {
5217 "Invalid fc mode; valid modes are 0 through 3\n");
5221 /* Set fc ability for port */
5222 hw->fc.requested_mode = requested_fc;
5223 aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
5226 "%s: Error setting new fc mode %d; fc_err %#x\n",
5227 __func__, aq_error, fc_aq_err);
5230 pf->fc = requested_fc;
5232 /* Get new link state */
5233 i40e_msec_delay(250);
5234 hw->phy.get_link_info = TRUE;
5235 i40e_get_link_status(hw, &pf->link_up);
5241 ixl_current_speed(SYSCTL_HANDLER_ARGS)
5243 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5244 struct i40e_hw *hw = &pf->hw;
5245 int error = 0, index = 0;
5256 ixl_update_link_status(pf);
5258 switch (hw->phy.link_info.link_speed) {
5259 case I40E_LINK_SPEED_100MB:
5262 case I40E_LINK_SPEED_1GB:
5265 case I40E_LINK_SPEED_10GB:
5268 case I40E_LINK_SPEED_40GB:
5271 case I40E_LINK_SPEED_20GB:
5274 case I40E_LINK_SPEED_UNKNOWN:
5280 error = sysctl_handle_string(oidp, speeds[index],
5281 strlen(speeds[index]), req);
5286 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
5288 struct i40e_hw *hw = &pf->hw;
5289 device_t dev = pf->dev;
5290 struct i40e_aq_get_phy_abilities_resp abilities;
5291 struct i40e_aq_set_phy_config config;
5292 enum i40e_status_code aq_error = 0;
5294 /* Get current capability information */
5295 aq_error = i40e_aq_get_phy_capabilities(hw,
5296 FALSE, FALSE, &abilities, NULL);
5299 "%s: Error getting phy capabilities %d,"
5300 " aq error: %d\n", __func__, aq_error,
5301 hw->aq.asq_last_status);
5305 /* Prepare new config */
5306 bzero(&config, sizeof(config));
5307 config.phy_type = abilities.phy_type;
5308 config.abilities = abilities.abilities
5309 | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
5310 config.eee_capability = abilities.eee_capability;
5311 config.eeer = abilities.eeer_val;
5312 config.low_power_ctrl = abilities.d3_lpan;
5313 /* Translate into aq cmd link_speed */
5315 config.link_speed |= I40E_LINK_SPEED_40GB;
5317 config.link_speed |= I40E_LINK_SPEED_20GB;
5319 config.link_speed |= I40E_LINK_SPEED_10GB;
5321 config.link_speed |= I40E_LINK_SPEED_1GB;
5323 config.link_speed |= I40E_LINK_SPEED_100MB;
5325 /* Do aq command & restart link */
5326 aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
5329 "%s: Error setting new phy config %d,"
5330 " aq error: %d\n", __func__, aq_error,
5331 hw->aq.asq_last_status);
5336 ** This seems a bit heavy handed, but we
5337 ** need to get a reinit on some devices
5340 ixl_stop_locked(pf);
5341 ixl_init_locked(pf);
5348 ** Control link advertise speed:
5350 ** 0x1 - advertise 100 Mb
5351 ** 0x2 - advertise 1G
5352 ** 0x4 - advertise 10G
5353 ** 0x8 - advertise 20G
5354 ** 0x10 - advertise 40G
5356 ** Set to 0 to disable link
5359 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
5361 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5362 struct i40e_hw *hw = &pf->hw;
5363 device_t dev = pf->dev;
5364 int requested_ls = 0;
5367 /* Read in new mode */
5368 requested_ls = pf->advertised_speed;
5369 error = sysctl_handle_int(oidp, &requested_ls, 0, req);
5370 if ((error) || (req->newptr == NULL))
5372 /* Check for sane value */
5373 if (requested_ls > 0x10) {
5374 device_printf(dev, "Invalid advertised speed; "
5375 "valid modes are 0x1 through 0x10\n");
5378 /* Then check for validity based on adapter type */
5379 switch (hw->device_id) {
5380 case I40E_DEV_ID_10G_BASE_T:
5381 case I40E_DEV_ID_10G_BASE_T4:
5383 if (requested_ls & ~(0x7)) {
5385 "Only 100M/1G/10G speeds supported on this device.\n");
5389 case I40E_DEV_ID_20G_KR2:
5390 case I40E_DEV_ID_20G_KR2_A:
5392 if (requested_ls & ~(0xE)) {
5394 "Only 1G/10G/20G speeds supported on this device.\n");
5398 case I40E_DEV_ID_KX_B:
5399 case I40E_DEV_ID_QSFP_A:
5400 case I40E_DEV_ID_QSFP_B:
5402 if (requested_ls & ~(0x10)) {
5404 "Only 40G speeds supported on this device.\n");
5410 if (requested_ls & ~(0x6)) {
5412 "Only 1/10Gbs speeds are supported on this device.\n");
5418 /* Exit if no change */
5419 if (pf->advertised_speed == requested_ls)
5422 error = ixl_set_advertised_speeds(pf, requested_ls);
5426 pf->advertised_speed = requested_ls;
5427 ixl_update_link_status(pf);
5432 ** Get the width and transaction speed of
5433 ** the bus this adapter is plugged into.
5436 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
5441 /* Get the PCI Express Capabilities offset */
5442 pci_find_cap(dev, PCIY_EXPRESS, &offset);
5444 /* ...and read the Link Status Register */
5445 link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
5447 switch (link & I40E_PCI_LINK_WIDTH) {
5448 case I40E_PCI_LINK_WIDTH_1:
5449 hw->bus.width = i40e_bus_width_pcie_x1;
5451 case I40E_PCI_LINK_WIDTH_2:
5452 hw->bus.width = i40e_bus_width_pcie_x2;
5454 case I40E_PCI_LINK_WIDTH_4:
5455 hw->bus.width = i40e_bus_width_pcie_x4;
5457 case I40E_PCI_LINK_WIDTH_8:
5458 hw->bus.width = i40e_bus_width_pcie_x8;
5461 hw->bus.width = i40e_bus_width_unknown;
5465 switch (link & I40E_PCI_LINK_SPEED) {
5466 case I40E_PCI_LINK_SPEED_2500:
5467 hw->bus.speed = i40e_bus_speed_2500;
5469 case I40E_PCI_LINK_SPEED_5000:
5470 hw->bus.speed = i40e_bus_speed_5000;
5472 case I40E_PCI_LINK_SPEED_8000:
5473 hw->bus.speed = i40e_bus_speed_8000;
5476 hw->bus.speed = i40e_bus_speed_unknown;
5480 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
5481 ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
5482 (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
5483 (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
5484 (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
5485 (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
5486 (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
5489 if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
5490 (hw->bus.speed < i40e_bus_speed_8000)) {
5491 device_printf(dev, "PCI-Express bandwidth available"
5492 " for this device\n may be insufficient for"
5493 " optimal performance.\n");
5494 device_printf(dev, "For expected performance a x8 "
5495 "PCIE Gen3 slot is required.\n");
5502 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
5504 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5505 struct i40e_hw *hw = &pf->hw;
5508 sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5509 ixl_nvm_version_str(hw, sbuf);
5518 ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma)
5520 if ((nvma->command == I40E_NVM_READ) &&
5521 ((nvma->config & 0xFF) == 0xF) &&
5522 (((nvma->config & 0xF00) >> 8) == 0xF) &&
5523 (nvma->offset == 0) &&
5524 (nvma->data_size == 1)) {
5525 // device_printf(dev, "- Get Driver Status Command\n");
5527 else if (nvma->command == I40E_NVM_READ) {
5531 switch (nvma->command) {
5533 device_printf(dev, "- command: I40E_NVM_READ\n");
5536 device_printf(dev, "- command: I40E_NVM_WRITE\n");
5539 device_printf(dev, "- command: unknown 0x%08x\n", nvma->command);
5543 device_printf(dev, "- config (ptr) : 0x%02x\n", nvma->config & 0xFF);
5544 device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8);
5545 device_printf(dev, "- offset : 0x%08x\n", nvma->offset);
5546 device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size);
5552 ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd)
5554 struct i40e_hw *hw = &pf->hw;
5555 struct i40e_nvm_access *nvma;
5556 device_t dev = pf->dev;
5557 enum i40e_status_code status = 0;
5560 DEBUGFUNC("ixl_handle_nvmupd_cmd");
5563 if (ifd->ifd_len < sizeof(struct i40e_nvm_access) ||
5564 ifd->ifd_data == NULL) {
5565 device_printf(dev, "%s: incorrect ifdrv length or data pointer\n",
5567 device_printf(dev, "%s: ifdrv length: %lu, sizeof(struct i40e_nvm_access): %lu\n",
5568 __func__, ifd->ifd_len, sizeof(struct i40e_nvm_access));
5569 device_printf(dev, "%s: data pointer: %p\n", __func__,
5574 nvma = (struct i40e_nvm_access *)ifd->ifd_data;
5577 ixl_print_nvm_cmd(dev, nvma);
5580 if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
5582 while (count++ < 100) {
5583 i40e_msec_delay(100);
5584 if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING))
5589 if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) {
5591 status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno);
5598 device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n",
5602 * -EPERM is actually ERESTART, which the kernel interprets as it needing
5603 * to run this ioctl again. So use -EACCES for -EPERM instead.
5605 if (perrno == -EPERM)
5611 #ifdef IXL_DEBUG_SYSCTL
5613 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
5615 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5616 struct i40e_hw *hw = &pf->hw;
5617 struct i40e_link_status link_status;
5620 enum i40e_status_code aq_error = 0;
5622 aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
5624 printf("i40e_aq_get_link_info() error %d\n", aq_error);
5629 "PHY Type : %#04x\n"
5631 "Link info: %#04x\n"
5633 "Ext info : %#04x\n"
5637 link_status.phy_type, link_status.link_speed,
5638 link_status.link_info, link_status.an_info,
5639 link_status.ext_info, link_status.max_frame_size,
5640 link_status.pacing, link_status.crc_enable);
5642 return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5646 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
5648 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5649 struct i40e_hw *hw = &pf->hw;
5651 enum i40e_status_code aq_error = 0;
5653 struct i40e_aq_get_phy_abilities_resp abilities;
5655 aq_error = i40e_aq_get_phy_capabilities(hw,
5656 TRUE, FALSE, &abilities, NULL);
5658 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
5663 "PHY Type : %#010x\n"
5665 "Abilities: %#04x\n"
5667 "EEER reg : %#010x\n"
5669 abilities.phy_type, abilities.link_speed,
5670 abilities.abilities, abilities.eee_capability,
5671 abilities.eeer_val, abilities.d3_lpan);
5673 return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5677 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5679 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5680 struct ixl_vsi *vsi = &pf->vsi;
5681 struct ixl_mac_filter *f;
5686 int ftl_counter = 0;
5690 SLIST_FOREACH(f, &vsi->ftl, next) {
5695 sysctl_handle_string(oidp, "(none)", 6, req);
5699 buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5700 buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5702 sprintf(buf_i++, "\n");
5703 SLIST_FOREACH(f, &vsi->ftl, next) {
5705 MAC_FORMAT ", vlan %4d, flags %#06x",
5706 MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5708 /* don't print '\n' for last entry */
5709 if (++ftl_counter != ftl_len) {
5710 sprintf(buf_i, "\n");
5715 error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5717 printf("sysctl error: %d\n", error);
5718 free(buf, M_DEVBUF);
5722 #define IXL_SW_RES_SIZE 0x14
5724 ixl_res_alloc_cmp(const void *a, const void *b)
5726 const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5727 one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5728 two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5730 return ((int)one->resource_type - (int)two->resource_type);
5734 * Longest string length: 25
5737 ixl_switch_res_type_string(u8 type)
5739 static char * ixl_switch_res_type_strings[0x14] = {
5742 "Perfect Match MAC address",
5745 "Multicast hash entry",
5746 "Unicast hash entry",
5750 "VLAN Statistic Pool",
5753 "Inner VLAN Forward filter",
5763 return ixl_switch_res_type_strings[type];
5765 return "(Reserved)";
5769 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5771 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5772 struct i40e_hw *hw = &pf->hw;
5773 device_t dev = pf->dev;
5778 struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5780 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5782 device_printf(dev, "Could not allocate sbuf for output.\n");
5786 bzero(resp, sizeof(resp));
5787 error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5793 "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5794 __func__, error, hw->aq.asq_last_status);
5799 /* Sort entries by type for display */
5800 qsort(resp, num_entries,
5801 sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5802 &ixl_res_alloc_cmp);
5804 sbuf_cat(buf, "\n");
5805 sbuf_printf(buf, "# of entries: %d\n", num_entries);
5807 " Type | Guaranteed | Total | Used | Un-allocated\n"
5808 " | (this) | (all) | (this) | (all) \n");
5809 for (int i = 0; i < num_entries; i++) {
5811 "%25s | %10d %5d %6d %12d",
5812 ixl_switch_res_type_string(resp[i].resource_type),
5816 resp[i].total_unalloced);
5817 if (i < num_entries - 1)
5818 sbuf_cat(buf, "\n");
5821 error = sbuf_finish(buf);
5823 device_printf(dev, "Error finishing sbuf: %d\n", error);
5830 ** Caller must init and delete sbuf; this function will clear and
5831 ** finish it for caller.
5833 ** XXX: Cannot use the SEID for this, since there is no longer a
5834 ** fixed mapping between SEID and element type.
5837 ixl_switch_element_string(struct sbuf *s,
5838 struct i40e_aqc_switch_config_element_resp *element)
5842 switch (element->element_type) {
5843 case I40E_AQ_SW_ELEM_TYPE_MAC:
5844 sbuf_printf(s, "MAC %3d", element->element_info);
5846 case I40E_AQ_SW_ELEM_TYPE_PF:
5847 sbuf_printf(s, "PF %3d", element->element_info);
5849 case I40E_AQ_SW_ELEM_TYPE_VF:
5850 sbuf_printf(s, "VF %3d", element->element_info);
5852 case I40E_AQ_SW_ELEM_TYPE_EMP:
5855 case I40E_AQ_SW_ELEM_TYPE_BMC:
5858 case I40E_AQ_SW_ELEM_TYPE_PV:
5861 case I40E_AQ_SW_ELEM_TYPE_VEB:
5864 case I40E_AQ_SW_ELEM_TYPE_PA:
5867 case I40E_AQ_SW_ELEM_TYPE_VSI:
5868 sbuf_printf(s, "VSI %3d", element->element_info);
5876 return sbuf_data(s);
5880 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5882 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5883 struct i40e_hw *hw = &pf->hw;
5884 device_t dev = pf->dev;
5889 u8 aq_buf[I40E_AQ_LARGE_BUF];
5891 struct i40e_aqc_get_switch_config_resp *sw_config;
5892 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5894 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5896 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5900 error = i40e_aq_get_switch_config(hw, sw_config,
5901 sizeof(aq_buf), &next, NULL);
5904 "%s: aq_get_switch_config() error %d, aq error %d\n",
5905 __func__, error, hw->aq.asq_last_status);
5910 device_printf(dev, "%s: TODO: get more config with SEID %d\n",
5913 nmbuf = sbuf_new_auto();
5915 device_printf(dev, "Could not allocate sbuf for name output.\n");
5920 sbuf_cat(buf, "\n");
5921 // Assuming <= 255 elements in switch
5922 sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported);
5923 sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total);
5925 ** Revision -- all elements are revision 1 for now
5928 "SEID ( Name ) | Uplink | Downlink | Conn Type\n"
5929 " | | | (uplink)\n");
5930 for (int i = 0; i < sw_config->header.num_reported; i++) {
5931 // "%4d (%8s) | %8s %8s %#8x",
5932 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5934 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5935 &sw_config->element[i]));
5936 sbuf_cat(buf, " | ");
5937 sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid);
5939 sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid);
5941 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5942 if (i < sw_config->header.num_reported - 1)
5943 sbuf_cat(buf, "\n");
5947 error = sbuf_finish(buf);
5949 device_printf(dev, "Error finishing sbuf: %d\n", error);
5957 ixl_debug_info(SYSCTL_HANDLER_ARGS)
5960 int error, input = 0;
5962 error = sysctl_handle_int(oidp, &input, 0, req);
5964 if (error || !req->newptr)
5968 pf = (struct ixl_pf *)arg1;
5969 ixl_print_debug_info(pf);
5976 ixl_print_debug_info(struct ixl_pf *pf)
5978 struct i40e_hw *hw = &pf->hw;
5979 struct ixl_vsi *vsi = &pf->vsi;
5980 struct ixl_queue *que = vsi->queues;
5981 struct rx_ring *rxr = &que->rxr;
5982 struct tx_ring *txr = &que->txr;
5986 printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
5987 printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
5988 printf("RX next check = %x\n", rxr->next_check);
5989 printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
5990 printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
5991 printf("TX desc avail = %x\n", txr->avail);
5993 reg = rd32(hw, I40E_GLV_GORCL(0xc));
5994 printf("RX Bytes = %x\n", reg);
5995 reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
5996 printf("Port RX Bytes = %x\n", reg);
5997 reg = rd32(hw, I40E_GLV_RDPC(0xc));
5998 printf("RX discard = %x\n", reg);
5999 reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
6000 printf("Port RX discard = %x\n", reg);
6002 reg = rd32(hw, I40E_GLV_TEPC(0xc));
6003 printf("TX errors = %x\n", reg);
6004 reg = rd32(hw, I40E_GLV_GOTCL(0xc));
6005 printf("TX Bytes = %x\n", reg);
6007 reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
6008 printf("RX undersize = %x\n", reg);
6009 reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
6010 printf("RX fragments = %x\n", reg);
6011 reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
6012 printf("RX oversize = %x\n", reg);
6013 reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
6014 printf("RX length error = %x\n", reg);
6015 reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
6016 printf("mac remote fault = %x\n", reg);
6017 reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
6018 printf("mac local fault = %x\n", reg);
6021 #endif /* IXL_DEBUG_SYSCTL */
6025 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6028 struct ixl_vsi *vsi;
6029 struct i40e_vsi_context vsi_ctx;
6031 uint16_t first_queue;
6032 enum i40e_status_code code;
6037 vsi_ctx.pf_num = hw->pf_id;
6038 vsi_ctx.uplink_seid = pf->veb_seid;
6039 vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
6040 vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6041 vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
6043 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6045 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6046 vsi_ctx.info.switch_id = htole16(0);
6048 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
6049 vsi_ctx.info.sec_flags = 0;
6050 if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
6051 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
6053 /* TODO: If a port VLAN is set, then this needs to be changed */
6054 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6055 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6056 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
6058 vsi_ctx.info.valid_sections |=
6059 htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
6060 vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
6061 first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
6062 for (i = 0; i < IXLV_MAX_QUEUES; i++)
6063 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
6064 for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
6065 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
6067 vsi_ctx.info.tc_mapping[0] = htole16(
6068 (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6069 (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
6071 code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
6072 if (code != I40E_SUCCESS)
6073 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6074 vf->vsi.seid = vsi_ctx.seid;
6075 vf->vsi.vsi_num = vsi_ctx.vsi_number;
6076 vf->vsi.first_queue = first_queue;
6077 vf->vsi.num_queues = IXLV_MAX_QUEUES;
6079 code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
6080 if (code != I40E_SUCCESS)
6081 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6083 code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
6084 if (code != I40E_SUCCESS) {
6085 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
6086 ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6087 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6090 memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
6095 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6102 error = ixl_vf_alloc_vsi(pf, vf);
6106 vf->vsi.hw_filters_add = 0;
6107 vf->vsi.hw_filters_del = 0;
6108 ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
6109 ixl_reconfigure_filters(&vf->vsi);
6115 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
6122 * Two queues are mapped in a single register, so we have to do some
6123 * gymnastics to convert the queue number into a register index and
6127 shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
6129 qtable = i40e_read_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
6130 qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
6131 qtable |= val << shift;
6132 i40e_write_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
6136 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
6145 * Contiguous mappings aren't actually supported by the hardware,
6146 * so we have to use non-contiguous mappings.
6148 i40e_write_rx_ctl(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
6149 I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
6151 wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
6152 I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
6154 for (i = 0; i < vf->vsi.num_queues; i++) {
6155 qtable = (vf->vsi.first_queue + i) <<
6156 I40E_VPLAN_QTABLE_QINDEX_SHIFT;
6158 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
6161 /* Map queues allocated to VF to its VSI. */
6162 for (i = 0; i < vf->vsi.num_queues; i++)
6163 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
6165 /* Set rest of VSI queues as unused. */
6166 for (; i < IXL_MAX_VSI_QUEUES; i++)
6167 ixl_vf_map_vsi_queue(hw, vf, i,
6168 I40E_VSILAN_QTABLE_QINDEX_0_MASK);
6174 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
6183 i40e_aq_delete_element(hw, vsi->seid, NULL);
6187 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
6190 wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
6195 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
6198 wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
6199 I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
6204 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
6207 uint32_t vfint_reg, vpint_reg;
6212 ixl_vf_vsi_release(pf, &vf->vsi);
6214 /* Index 0 has a special register. */
6215 ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
6217 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6218 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
6219 ixl_vf_disable_queue_intr(hw, vfint_reg);
6222 /* Index 0 has a special register. */
6223 ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
6225 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6226 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
6227 ixl_vf_unregister_intr(hw, vpint_reg);
6230 vf->vsi.num_queues = 0;
6234 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
6238 uint16_t global_vf_num;
6242 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6244 wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
6245 (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
6246 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6247 ciad = rd32(hw, I40E_PF_PCI_CIAD);
6248 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
6257 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6264 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6265 vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6266 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6269 ixl_reinit_vf(pf, vf);
6273 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6276 uint32_t vfrstat, vfrtrig;
6281 error = ixl_flush_pcie(pf, vf);
6283 device_printf(pf->dev,
6284 "Timed out waiting for PCIe activity to stop on VF-%d\n",
6287 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6290 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
6291 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
6295 if (i == IXL_VF_RESET_TIMEOUT)
6296 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
6298 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
6300 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6301 vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6302 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6304 if (vf->vsi.seid != 0)
6305 ixl_disable_rings(&vf->vsi);
6307 ixl_vf_release_resources(pf, vf);
6308 ixl_vf_setup_vsi(pf, vf);
6309 ixl_vf_map_queues(pf, vf);
6311 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
6316 ixl_vc_opcode_str(uint16_t op)
6320 case I40E_VIRTCHNL_OP_VERSION:
6322 case I40E_VIRTCHNL_OP_RESET_VF:
6323 return ("RESET_VF");
6324 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6325 return ("GET_VF_RESOURCES");
6326 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6327 return ("CONFIG_TX_QUEUE");
6328 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6329 return ("CONFIG_RX_QUEUE");
6330 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6331 return ("CONFIG_VSI_QUEUES");
6332 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6333 return ("CONFIG_IRQ_MAP");
6334 case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6335 return ("ENABLE_QUEUES");
6336 case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6337 return ("DISABLE_QUEUES");
6338 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6339 return ("ADD_ETHER_ADDRESS");
6340 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6341 return ("DEL_ETHER_ADDRESS");
6342 case I40E_VIRTCHNL_OP_ADD_VLAN:
6343 return ("ADD_VLAN");
6344 case I40E_VIRTCHNL_OP_DEL_VLAN:
6345 return ("DEL_VLAN");
6346 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6347 return ("CONFIG_PROMISCUOUS_MODE");
6348 case I40E_VIRTCHNL_OP_GET_STATS:
6349 return ("GET_STATS");
6350 case I40E_VIRTCHNL_OP_FCOE:
6352 case I40E_VIRTCHNL_OP_EVENT:
6360 ixl_vc_opcode_level(uint16_t opcode)
6363 case I40E_VIRTCHNL_OP_GET_STATS:
6371 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6372 enum i40e_status_code status, void *msg, uint16_t len)
6378 global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
6380 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
6381 "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
6382 ixl_vc_opcode_str(op), op, status, vf->vf_num);
6384 i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
6388 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
6391 ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
6395 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6396 enum i40e_status_code status, const char *file, int line)
6399 I40E_VC_DEBUG(pf, 1,
6400 "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
6401 ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
6402 ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
6406 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6409 struct i40e_virtchnl_version_info reply;
6411 if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
6412 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
6417 vf->version = ((struct i40e_virtchnl_version_info *)msg)->minor;
6419 reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
6420 reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
6421 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
6426 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6430 if (msg_size != 0) {
6431 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
6436 ixl_reset_vf(pf, vf);
6438 /* No response to a reset message. */
6442 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6445 struct i40e_virtchnl_vf_resource reply;
6447 if ((vf->version == 0 && msg_size != 0) ||
6448 (vf->version == 1 && msg_size != 4)) {
6449 device_printf(pf->dev, "Invalid GET_VF_RESOURCES message size,"
6450 " for VF version %d.%d\n", I40E_VIRTCHNL_VERSION_MAJOR,
6452 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6457 bzero(&reply, sizeof(reply));
6459 if (vf->version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS)
6460 reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
6461 I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
6462 I40E_VIRTCHNL_VF_OFFLOAD_VLAN;
6464 reply.vf_offload_flags = *(u32 *)msg;
6467 reply.num_queue_pairs = vf->vsi.num_queues;
6468 reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
6469 reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
6470 reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
6471 reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
6472 memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
6474 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6475 I40E_SUCCESS, &reply, sizeof(reply));
6479 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6480 struct i40e_virtchnl_txq_info *info)
6483 struct i40e_hmc_obj_txq txq;
6484 uint16_t global_queue_num, global_vf_num;
6485 enum i40e_status_code status;
6489 global_queue_num = vf->vsi.first_queue + info->queue_id;
6490 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6491 bzero(&txq, sizeof(txq));
6493 status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
6494 if (status != I40E_SUCCESS)
6497 txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
6499 txq.head_wb_ena = info->headwb_enabled;
6500 txq.head_wb_addr = info->dma_headwb_addr;
6501 txq.qlen = info->ring_len;
6502 txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
6503 txq.rdylist_act = 0;
6505 status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
6506 if (status != I40E_SUCCESS)
6509 qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
6510 (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
6511 (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
6512 wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
6519 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6520 struct i40e_virtchnl_rxq_info *info)
6523 struct i40e_hmc_obj_rxq rxq;
6524 uint16_t global_queue_num;
6525 enum i40e_status_code status;
6528 global_queue_num = vf->vsi.first_queue + info->queue_id;
6529 bzero(&rxq, sizeof(rxq));
6531 if (info->databuffer_size > IXL_VF_MAX_BUFFER)
6534 if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
6535 info->max_pkt_size < ETHER_MIN_LEN)
6538 if (info->splithdr_enabled) {
6539 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
6542 rxq.hsplit_0 = info->rx_split_pos &
6543 (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
6544 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
6545 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
6546 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
6547 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
6552 status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
6553 if (status != I40E_SUCCESS)
6556 rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
6557 rxq.qlen = info->ring_len;
6559 rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
6565 rxq.rxmax = info->max_pkt_size;
6566 rxq.tphrdesc_ena = 1;
6567 rxq.tphwdesc_ena = 1;
6568 rxq.tphdata_ena = 1;
6569 rxq.tphhead_ena = 1;
6573 status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
6574 if (status != I40E_SUCCESS)
6581 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6584 struct i40e_virtchnl_vsi_queue_config_info *info;
6585 struct i40e_virtchnl_queue_pair_info *pair;
6588 if (msg_size < sizeof(*info)) {
6589 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6595 if (info->num_queue_pairs == 0) {
6596 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6601 if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
6602 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6607 if (info->vsi_id != vf->vsi.vsi_num) {
6608 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6613 for (i = 0; i < info->num_queue_pairs; i++) {
6614 pair = &info->qpair[i];
6616 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
6617 pair->rxq.vsi_id != vf->vsi.vsi_num ||
6618 pair->txq.queue_id != pair->rxq.queue_id ||
6619 pair->txq.queue_id >= vf->vsi.num_queues) {
6621 i40e_send_vf_nack(pf, vf,
6622 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6626 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
6627 i40e_send_vf_nack(pf, vf,
6628 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6632 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
6633 i40e_send_vf_nack(pf, vf,
6634 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6639 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
6643 ixl_vf_set_qctl(struct ixl_pf *pf,
6644 const struct i40e_virtchnl_vector_map *vector,
6645 enum i40e_queue_type cur_type, uint16_t cur_queue,
6646 enum i40e_queue_type *last_type, uint16_t *last_queue)
6648 uint32_t offset, qctl;
6651 if (cur_type == I40E_QUEUE_TYPE_RX) {
6652 offset = I40E_QINT_RQCTL(cur_queue);
6653 itr_indx = vector->rxitr_idx;
6655 offset = I40E_QINT_TQCTL(cur_queue);
6656 itr_indx = vector->txitr_idx;
6659 qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
6660 (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
6661 (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
6662 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
6663 (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
6665 wr32(&pf->hw, offset, qctl);
6667 *last_type = cur_type;
6668 *last_queue = cur_queue;
6672 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
6673 const struct i40e_virtchnl_vector_map *vector)
6677 enum i40e_queue_type type, last_type;
6678 uint32_t lnklst_reg;
6679 uint16_t rxq_map, txq_map, cur_queue, last_queue;
6683 rxq_map = vector->rxq_map;
6684 txq_map = vector->txq_map;
6686 last_queue = IXL_END_OF_INTR_LNKLST;
6687 last_type = I40E_QUEUE_TYPE_RX;
6690 * The datasheet says to optimize performance, RX queues and TX queues
6691 * should be interleaved in the interrupt linked list, so we process
6692 * both at once here.
6694 while ((rxq_map != 0) || (txq_map != 0)) {
6696 qindex = ffs(txq_map) - 1;
6697 type = I40E_QUEUE_TYPE_TX;
6698 cur_queue = vf->vsi.first_queue + qindex;
6699 ixl_vf_set_qctl(pf, vector, type, cur_queue,
6700 &last_type, &last_queue);
6701 txq_map &= ~(1 << qindex);
6705 qindex = ffs(rxq_map) - 1;
6706 type = I40E_QUEUE_TYPE_RX;
6707 cur_queue = vf->vsi.first_queue + qindex;
6708 ixl_vf_set_qctl(pf, vector, type, cur_queue,
6709 &last_type, &last_queue);
6710 rxq_map &= ~(1 << qindex);
6714 if (vector->vector_id == 0)
6715 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
6717 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
6719 wr32(hw, lnklst_reg,
6720 (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
6721 (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
6727 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6730 struct i40e_virtchnl_irq_map_info *map;
6731 struct i40e_virtchnl_vector_map *vector;
6733 int i, largest_txq, largest_rxq;
6737 if (msg_size < sizeof(*map)) {
6738 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6744 if (map->num_vectors == 0) {
6745 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6750 if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
6751 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6756 for (i = 0; i < map->num_vectors; i++) {
6757 vector = &map->vecmap[i];
6759 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
6760 vector->vsi_id != vf->vsi.vsi_num) {
6761 i40e_send_vf_nack(pf, vf,
6762 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
6766 if (vector->rxq_map != 0) {
6767 largest_rxq = fls(vector->rxq_map) - 1;
6768 if (largest_rxq >= vf->vsi.num_queues) {
6769 i40e_send_vf_nack(pf, vf,
6770 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6776 if (vector->txq_map != 0) {
6777 largest_txq = fls(vector->txq_map) - 1;
6778 if (largest_txq >= vf->vsi.num_queues) {
6779 i40e_send_vf_nack(pf, vf,
6780 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6786 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6787 vector->txitr_idx > IXL_MAX_ITR_IDX) {
6788 i40e_send_vf_nack(pf, vf,
6789 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6794 ixl_vf_config_vector(pf, vf, vector);
6797 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6801 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6804 struct i40e_virtchnl_queue_select *select;
6807 if (msg_size != sizeof(*select)) {
6808 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6814 if (select->vsi_id != vf->vsi.vsi_num ||
6815 select->rx_queues == 0 || select->tx_queues == 0) {
6816 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6821 error = ixl_enable_rings(&vf->vsi);
6823 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6828 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6832 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6833 void *msg, uint16_t msg_size)
6835 struct i40e_virtchnl_queue_select *select;
6838 if (msg_size != sizeof(*select)) {
6839 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6845 if (select->vsi_id != vf->vsi.vsi_num ||
6846 select->rx_queues == 0 || select->tx_queues == 0) {
6847 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6852 error = ixl_disable_rings(&vf->vsi);
6854 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6859 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6863 ixl_zero_mac(const uint8_t *addr)
6865 uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6867 return (cmp_etheraddr(addr, zero));
6871 ixl_bcast_mac(const uint8_t *addr)
6874 return (cmp_etheraddr(addr, ixl_bcast_addr));
6878 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6881 if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6885 * If the VF is not allowed to change its MAC address, don't let it
6886 * set a MAC filter for an address that is not a multicast address and
6887 * is not its assigned MAC.
6889 if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6890 !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6897 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6900 struct i40e_virtchnl_ether_addr_list *addr_list;
6901 struct i40e_virtchnl_ether_addr *addr;
6902 struct ixl_vsi *vsi;
6904 size_t expected_size;
6908 if (msg_size < sizeof(*addr_list)) {
6909 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6915 expected_size = sizeof(*addr_list) +
6916 addr_list->num_elements * sizeof(*addr);
6918 if (addr_list->num_elements == 0 ||
6919 addr_list->vsi_id != vsi->vsi_num ||
6920 msg_size != expected_size) {
6921 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6926 for (i = 0; i < addr_list->num_elements; i++) {
6927 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6928 i40e_send_vf_nack(pf, vf,
6929 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6934 for (i = 0; i < addr_list->num_elements; i++) {
6935 addr = &addr_list->list[i];
6936 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6939 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6943 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6946 struct i40e_virtchnl_ether_addr_list *addr_list;
6947 struct i40e_virtchnl_ether_addr *addr;
6948 size_t expected_size;
6951 if (msg_size < sizeof(*addr_list)) {
6952 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6958 expected_size = sizeof(*addr_list) +
6959 addr_list->num_elements * sizeof(*addr);
6961 if (addr_list->num_elements == 0 ||
6962 addr_list->vsi_id != vf->vsi.vsi_num ||
6963 msg_size != expected_size) {
6964 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6969 for (i = 0; i < addr_list->num_elements; i++) {
6970 addr = &addr_list->list[i];
6971 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6972 i40e_send_vf_nack(pf, vf,
6973 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6978 for (i = 0; i < addr_list->num_elements; i++) {
6979 addr = &addr_list->list[i];
6980 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6983 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6986 static enum i40e_status_code
6987 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6989 struct i40e_vsi_context vsi_ctx;
6991 vsi_ctx.seid = vf->vsi.seid;
6993 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6994 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6995 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6996 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6997 return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
7001 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7004 struct i40e_virtchnl_vlan_filter_list *filter_list;
7005 enum i40e_status_code code;
7006 size_t expected_size;
7009 if (msg_size < sizeof(*filter_list)) {
7010 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7016 expected_size = sizeof(*filter_list) +
7017 filter_list->num_elements * sizeof(uint16_t);
7018 if (filter_list->num_elements == 0 ||
7019 filter_list->vsi_id != vf->vsi.vsi_num ||
7020 msg_size != expected_size) {
7021 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7026 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7027 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7032 for (i = 0; i < filter_list->num_elements; i++) {
7033 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7034 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7040 code = ixl_vf_enable_vlan_strip(pf, vf);
7041 if (code != I40E_SUCCESS) {
7042 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7046 for (i = 0; i < filter_list->num_elements; i++)
7047 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7049 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
7053 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7056 struct i40e_virtchnl_vlan_filter_list *filter_list;
7058 size_t expected_size;
7060 if (msg_size < sizeof(*filter_list)) {
7061 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7067 expected_size = sizeof(*filter_list) +
7068 filter_list->num_elements * sizeof(uint16_t);
7069 if (filter_list->num_elements == 0 ||
7070 filter_list->vsi_id != vf->vsi.vsi_num ||
7071 msg_size != expected_size) {
7072 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7077 for (i = 0; i < filter_list->num_elements; i++) {
7078 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7079 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7085 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7086 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7091 for (i = 0; i < filter_list->num_elements; i++)
7092 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7094 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
7098 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
7099 void *msg, uint16_t msg_size)
7101 struct i40e_virtchnl_promisc_info *info;
7102 enum i40e_status_code code;
7104 if (msg_size != sizeof(*info)) {
7105 i40e_send_vf_nack(pf, vf,
7106 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7110 if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) {
7111 i40e_send_vf_nack(pf, vf,
7112 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7117 if (info->vsi_id != vf->vsi.vsi_num) {
7118 i40e_send_vf_nack(pf, vf,
7119 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7123 code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
7124 info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
7125 if (code != I40E_SUCCESS) {
7126 i40e_send_vf_nack(pf, vf,
7127 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7131 code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
7132 info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
7133 if (code != I40E_SUCCESS) {
7134 i40e_send_vf_nack(pf, vf,
7135 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7139 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
7143 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7146 struct i40e_virtchnl_queue_select *queue;
7148 if (msg_size != sizeof(*queue)) {
7149 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7155 if (queue->vsi_id != vf->vsi.vsi_num) {
7156 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7161 ixl_update_eth_stats(&vf->vsi);
7163 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7164 I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
7168 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
7172 uint16_t vf_num, msg_size;
7175 vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
7176 opcode = le32toh(event->desc.cookie_high);
7178 if (vf_num >= pf->num_vfs) {
7179 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
7183 vf = &pf->vfs[vf_num];
7184 msg = event->msg_buf;
7185 msg_size = event->msg_len;
7187 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
7188 "Got msg %s(%d) from VF-%d of size %d\n",
7189 ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
7192 case I40E_VIRTCHNL_OP_VERSION:
7193 ixl_vf_version_msg(pf, vf, msg, msg_size);
7195 case I40E_VIRTCHNL_OP_RESET_VF:
7196 ixl_vf_reset_msg(pf, vf, msg, msg_size);
7198 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
7199 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
7201 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
7202 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
7204 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
7205 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
7207 case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
7208 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
7210 case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
7211 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
7213 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
7214 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
7216 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
7217 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
7219 case I40E_VIRTCHNL_OP_ADD_VLAN:
7220 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
7222 case I40E_VIRTCHNL_OP_DEL_VLAN:
7223 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
7225 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
7226 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
7228 case I40E_VIRTCHNL_OP_GET_STATS:
7229 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
7232 /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
7233 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
7234 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
7236 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
7241 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
7243 ixl_handle_vflr(void *arg, int pending)
7247 uint16_t global_vf_num;
7248 uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
7255 for (i = 0; i < pf->num_vfs; i++) {
7256 global_vf_num = hw->func_caps.vf_base_id + i;
7258 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
7259 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
7260 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
7261 if (vflrstat & vflrstat_mask) {
7262 wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
7265 ixl_reinit_vf(pf, &pf->vfs[i]);
7269 icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
7270 icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
7271 wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
7278 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
7282 case I40E_AQ_RC_EPERM:
7284 case I40E_AQ_RC_ENOENT:
7286 case I40E_AQ_RC_ESRCH:
7288 case I40E_AQ_RC_EINTR:
7290 case I40E_AQ_RC_EIO:
7292 case I40E_AQ_RC_ENXIO:
7294 case I40E_AQ_RC_E2BIG:
7296 case I40E_AQ_RC_EAGAIN:
7298 case I40E_AQ_RC_ENOMEM:
7300 case I40E_AQ_RC_EACCES:
7302 case I40E_AQ_RC_EFAULT:
7304 case I40E_AQ_RC_EBUSY:
7306 case I40E_AQ_RC_EEXIST:
7308 case I40E_AQ_RC_EINVAL:
7310 case I40E_AQ_RC_ENOTTY:
7312 case I40E_AQ_RC_ENOSPC:
7314 case I40E_AQ_RC_ENOSYS:
7316 case I40E_AQ_RC_ERANGE:
7318 case I40E_AQ_RC_EFLUSHED:
7319 return (EINVAL); /* No exact equivalent in errno.h */
7320 case I40E_AQ_RC_BAD_ADDR:
7322 case I40E_AQ_RC_EMODE:
7324 case I40E_AQ_RC_EFBIG:
7332 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
7336 struct ixl_vsi *pf_vsi;
7337 enum i40e_status_code ret;
7340 pf = device_get_softc(dev);
7345 pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
7348 if (pf->vfs == NULL) {
7353 for (i = 0; i < num_vfs; i++)
7354 sysctl_ctx_init(&pf->vfs[i].ctx);
7356 ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
7357 1, FALSE, &pf->veb_seid, FALSE, NULL);
7358 if (ret != I40E_SUCCESS) {
7359 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
7360 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
7365 // TODO: [Configure MSI-X here]
7366 ixl_enable_adminq(hw);
7368 pf->num_vfs = num_vfs;
7373 free(pf->vfs, M_IXL);
7380 ixl_iov_uninit(device_t dev)
7384 struct ixl_vsi *vsi;
7389 pf = device_get_softc(dev);
7395 for (i = 0; i < pf->num_vfs; i++) {
7396 if (pf->vfs[i].vsi.seid != 0)
7397 i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
7400 if (pf->veb_seid != 0) {
7401 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
7405 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
7406 ixl_disable_intr(vsi);
7409 num_vfs = pf->num_vfs;
7415 /* Do this after the unlock as sysctl_ctx_free might sleep. */
7416 for (i = 0; i < num_vfs; i++)
7417 sysctl_ctx_free(&vfs[i].ctx);
7422 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
7424 char sysctl_name[QUEUE_NAME_LEN];
7431 pf = device_get_softc(dev);
7432 vf = &pf->vfs[vfnum];
7438 vf->vf_flags = VF_FLAG_ENABLED;
7439 SLIST_INIT(&vf->vsi.ftl);
7441 error = ixl_vf_setup_vsi(pf, vf);
7445 if (nvlist_exists_binary(params, "mac-addr")) {
7446 mac = nvlist_get_binary(params, "mac-addr", &size);
7447 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
7449 if (nvlist_get_bool(params, "allow-set-mac"))
7450 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7453 * If the administrator has not specified a MAC address then
7454 * we must allow the VF to choose one.
7456 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7458 if (nvlist_get_bool(params, "mac-anti-spoof"))
7459 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
7461 if (nvlist_get_bool(params, "allow-promisc"))
7462 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
7464 /* TODO: Get VLAN that PF has set for the VF */
7466 vf->vf_flags |= VF_FLAG_VLAN_CAP;
7468 ixl_reset_vf(pf, vf);
7472 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
7473 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
7478 #endif /* PCI_IOV */