1 /******************************************************************************
3 Copyright (c) 2013-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
35 #ifndef IXL_STANDALONE_BUILD
37 #include "opt_inet6.h"
45 #include <net/rss_config.h>
48 /*********************************************************************
50 *********************************************************************/
51 char ixl_driver_version[] = "1.4.27-k";
53 /*********************************************************************
56 * Used by probe to select devices to load on
57 * Last field stores an index into ixl_strings
58 * Last entry must be all 0s
60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61 *********************************************************************/
63 static ixl_vendor_info_t ixl_vendor_info_array[] =
65 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
66 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
67 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
68 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
69 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
70 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
71 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
72 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
73 /* required last entry */
77 /*********************************************************************
78 * Table of branding strings
79 *********************************************************************/
81 static char *ixl_strings[] = {
82 "Intel(R) Ethernet Connection XL710 Driver"
86 /*********************************************************************
88 *********************************************************************/
89 static int ixl_probe(device_t);
90 static int ixl_attach(device_t);
91 static int ixl_detach(device_t);
92 static int ixl_shutdown(device_t);
93 static int ixl_get_hw_capabilities(struct ixl_pf *);
94 static void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
95 static int ixl_ioctl(struct ifnet *, u_long, caddr_t);
96 static void ixl_init(void *);
97 static void ixl_init_locked(struct ixl_pf *);
98 static void ixl_stop(struct ixl_pf *);
99 static void ixl_stop_locked(struct ixl_pf *);
100 static void ixl_media_status(struct ifnet *, struct ifmediareq *);
101 static int ixl_media_change(struct ifnet *);
102 static void ixl_update_link_status(struct ixl_pf *);
103 static int ixl_allocate_pci_resources(struct ixl_pf *);
104 static u16 ixl_get_bus_info(struct i40e_hw *, device_t);
105 static int ixl_setup_stations(struct ixl_pf *);
106 static int ixl_switch_config(struct ixl_pf *);
107 static int ixl_initialize_vsi(struct ixl_vsi *);
109 static int ixl_setup_adminq_msix(struct ixl_pf *);
110 static int ixl_setup_adminq_tq(struct ixl_pf *);
111 static int ixl_setup_queue_msix(struct ixl_vsi *);
112 static int ixl_setup_queue_tqs(struct ixl_vsi *);
113 static int ixl_teardown_adminq_msix(struct ixl_pf *);
114 static int ixl_teardown_queue_msix(struct ixl_vsi *);
115 static void ixl_configure_intr0_msix(struct ixl_pf *);
116 static void ixl_configure_queue_intr_msix(struct ixl_pf *);
117 static void ixl_free_queue_tqs(struct ixl_vsi *);
118 static void ixl_free_adminq_tq(struct ixl_pf *);
120 static int ixl_assign_vsi_legacy(struct ixl_pf *);
121 static int ixl_init_msix(struct ixl_pf *);
122 static void ixl_configure_itr(struct ixl_pf *);
123 static void ixl_configure_legacy(struct ixl_pf *);
124 static void ixl_free_pci_resources(struct ixl_pf *);
125 static void ixl_local_timer(void *);
126 static int ixl_setup_interface(device_t, struct ixl_vsi *);
127 static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
128 static void ixl_config_rss(struct ixl_vsi *);
129 static void ixl_set_queue_rx_itr(struct ixl_queue *);
130 static void ixl_set_queue_tx_itr(struct ixl_queue *);
131 static int ixl_set_advertised_speeds(struct ixl_pf *, int);
132 static void ixl_get_initial_advertised_speeds(struct ixl_pf *);
134 static int ixl_enable_rings(struct ixl_vsi *);
135 static int ixl_disable_rings(struct ixl_vsi *);
136 static void ixl_enable_intr(struct ixl_vsi *);
137 static void ixl_disable_intr(struct ixl_vsi *);
138 static void ixl_disable_rings_intr(struct ixl_vsi *);
140 static void ixl_enable_adminq(struct i40e_hw *);
141 static void ixl_disable_adminq(struct i40e_hw *);
142 static void ixl_enable_queue(struct i40e_hw *, int);
143 static void ixl_disable_queue(struct i40e_hw *, int);
144 static void ixl_enable_legacy(struct i40e_hw *);
145 static void ixl_disable_legacy(struct i40e_hw *);
147 static void ixl_set_promisc(struct ixl_vsi *);
148 static void ixl_add_multi(struct ixl_vsi *);
149 static void ixl_del_multi(struct ixl_vsi *);
150 static void ixl_register_vlan(void *, struct ifnet *, u16);
151 static void ixl_unregister_vlan(void *, struct ifnet *, u16);
152 static void ixl_setup_vlan_filters(struct ixl_vsi *);
154 static void ixl_init_filters(struct ixl_vsi *);
155 static void ixl_reconfigure_filters(struct ixl_vsi *vsi);
156 static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
157 static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
158 static void ixl_add_hw_filters(struct ixl_vsi *, int, int);
159 static void ixl_del_hw_filters(struct ixl_vsi *, int);
160 static struct ixl_mac_filter *
161 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
162 static void ixl_add_mc_filter(struct ixl_vsi *, u8 *);
163 static void ixl_free_mac_filters(struct ixl_vsi *vsi);
166 static void ixl_add_device_sysctls(struct ixl_pf *);
168 static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
169 static int ixl_set_advertise(SYSCTL_HANDLER_ARGS);
170 static int ixl_current_speed(SYSCTL_HANDLER_ARGS);
171 static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
173 #ifdef IXL_DEBUG_SYSCTL
174 static int ixl_debug_info(SYSCTL_HANDLER_ARGS);
175 static void ixl_print_debug_info(struct ixl_pf *);
177 static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
178 static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
179 static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
180 static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
181 static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
184 /* The MSI/X Interrupt handlers */
185 static void ixl_intr(void *);
186 static void ixl_msix_que(void *);
187 static void ixl_msix_adminq(void *);
188 static void ixl_handle_mdd_event(struct ixl_pf *);
190 /* Deferred interrupt tasklets */
191 static void ixl_do_adminq(void *, int);
194 static void ixl_add_hw_stats(struct ixl_pf *);
195 static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
196 struct sysctl_oid_list *, struct i40e_hw_port_stats *);
197 static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
198 struct sysctl_oid_list *,
199 struct i40e_eth_stats *);
200 static void ixl_update_stats_counters(struct ixl_pf *);
201 static void ixl_update_eth_stats(struct ixl_vsi *);
202 static void ixl_update_vsi_stats(struct ixl_vsi *);
203 static void ixl_pf_reset_stats(struct ixl_pf *);
204 static void ixl_vsi_reset_stats(struct ixl_vsi *);
205 static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
207 static void ixl_stat_update32(struct i40e_hw *, u32, bool,
210 static int ixl_handle_nvmupd_cmd(struct ixl_pf *, struct ifdrv *);
211 static void ixl_handle_empr_reset(struct ixl_pf *);
212 static int ixl_rebuild_hw_structs_after_reset(struct ixl_pf *);
214 /* Debug helper functions */
216 static void ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *);
220 static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
222 static int ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
223 static void ixl_iov_uninit(device_t dev);
224 static int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
226 static void ixl_handle_vf_msg(struct ixl_pf *,
227 struct i40e_arq_event_info *);
228 static void ixl_handle_vflr(void *arg, int pending);
230 static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
231 static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
234 /*********************************************************************
235 * FreeBSD Device Interface Entry Points
236 *********************************************************************/
238 static device_method_t ixl_methods[] = {
239 /* Device interface */
240 DEVMETHOD(device_probe, ixl_probe),
241 DEVMETHOD(device_attach, ixl_attach),
242 DEVMETHOD(device_detach, ixl_detach),
243 DEVMETHOD(device_shutdown, ixl_shutdown),
245 DEVMETHOD(pci_iov_init, ixl_iov_init),
246 DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
247 DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
252 static driver_t ixl_driver = {
253 "ixl", ixl_methods, sizeof(struct ixl_pf),
256 devclass_t ixl_devclass;
257 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
259 MODULE_DEPEND(ixl, pci, 1, 1, 1);
260 MODULE_DEPEND(ixl, ether, 1, 1, 1);
262 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
263 #endif /* DEV_NETMAP */
266 ** Global reset mutex
268 static struct mtx ixl_reset_mtx;
271 ** TUNEABLE PARAMETERS:
274 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
275 "IXL driver parameters");
278 * MSIX should be the default for best performance,
279 * but this allows it to be forced off for testing.
281 static int ixl_enable_msix = 1;
282 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
283 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
284 "Enable MSI-X interrupts");
287 ** Number of descriptors per ring:
288 ** - TX and RX are the same size
290 static int ixl_ringsz = DEFAULT_RING;
291 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
292 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
293 &ixl_ringsz, 0, "Descriptor Ring Size");
296 ** This can be set manually, if left as 0 the
297 ** number of queues will be calculated based
298 ** on cpus and msix vectors available.
300 int ixl_max_queues = 0;
301 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
302 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
303 &ixl_max_queues, 0, "Number of Queues");
306 ** Controls for Interrupt Throttling
307 ** - true/false for dynamic adjustment
308 ** - default values for static ITR
310 int ixl_dynamic_rx_itr = 1;
311 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
312 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
313 &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
315 int ixl_dynamic_tx_itr = 1;
316 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
317 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
318 &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
320 int ixl_rx_itr = IXL_ITR_8K;
321 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
322 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
323 &ixl_rx_itr, 0, "RX Interrupt Rate");
325 int ixl_tx_itr = IXL_ITR_4K;
326 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
327 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
328 &ixl_tx_itr, 0, "TX Interrupt Rate");
331 static int ixl_enable_fdir = 1;
332 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
333 /* Rate at which we sample */
334 int ixl_atr_rate = 20;
335 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
339 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
340 #include <dev/netmap/if_ixl_netmap.h>
341 #endif /* DEV_NETMAP */
343 static char *ixl_fc_string[6] = {
352 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
354 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
355 {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
357 /*********************************************************************
358 * Device identification routine
360 * ixl_probe determines if the driver should be loaded on
361 * the hardware based on PCI vendor/device id of the device.
363 * return BUS_PROBE_DEFAULT on success, positive on failure
364 *********************************************************************/
367 ixl_probe(device_t dev)
369 ixl_vendor_info_t *ent;
371 u16 pci_vendor_id, pci_device_id;
372 u16 pci_subvendor_id, pci_subdevice_id;
373 char device_name[256];
374 static bool lock_init = FALSE;
377 INIT_DEBUGOUT("ixl_probe: begin");
379 pci_vendor_id = pci_get_vendor(dev);
380 if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
383 pci_device_id = pci_get_device(dev);
384 pci_subvendor_id = pci_get_subvendor(dev);
385 pci_subdevice_id = pci_get_subdevice(dev);
387 ent = ixl_vendor_info_array;
388 while (ent->vendor_id != 0) {
389 if ((pci_vendor_id == ent->vendor_id) &&
390 (pci_device_id == ent->device_id) &&
392 ((pci_subvendor_id == ent->subvendor_id) ||
393 (ent->subvendor_id == 0)) &&
395 ((pci_subdevice_id == ent->subdevice_id) ||
396 (ent->subdevice_id == 0))) {
397 sprintf(device_name, "%s, Version - %s",
398 ixl_strings[ent->index],
400 device_set_desc_copy(dev, device_name);
401 /* One shot mutex init */
402 if (lock_init == FALSE) {
404 mtx_init(&ixl_reset_mtx,
406 "IXL RESET Lock", MTX_DEF);
408 return (BUS_PROBE_DEFAULT);
415 /*********************************************************************
416 * Device initialization routine
418 * The attach entry point is called when the driver is being loaded.
419 * This routine identifies the type of hardware, allocates all resources
420 * and initializes the hardware.
422 * return 0 on success, positive on failure
423 *********************************************************************/
426 ixl_attach(device_t dev)
434 nvlist_t *pf_schema, *vf_schema;
438 INIT_DEBUGOUT("ixl_attach: begin");
440 /* Allocate, clear, and link in our primary soft structure */
441 pf = device_get_softc(dev);
442 pf->dev = pf->osdep.dev = dev;
446 ** Note this assumes we have a single embedded VSI,
447 ** this could be enhanced later to allocate multiple
453 IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
455 /* Set up the timer callout */
456 callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
458 /* Save off the PCI information */
459 hw->vendor_id = pci_get_vendor(dev);
460 hw->device_id = pci_get_device(dev);
461 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
462 hw->subsystem_vendor_id =
463 pci_read_config(dev, PCIR_SUBVEND_0, 2);
464 hw->subsystem_device_id =
465 pci_read_config(dev, PCIR_SUBDEV_0, 2);
467 hw->bus.device = pci_get_slot(dev);
468 hw->bus.func = pci_get_function(dev);
470 pf->vc_debug_lvl = 1;
472 /* Do PCI setup - map BAR0, etc */
473 if (ixl_allocate_pci_resources(pf)) {
474 device_printf(dev, "Allocation of PCI resources failed\n");
479 /* Establish a clean starting point */
481 error = i40e_pf_reset(hw);
483 device_printf(dev, "PF reset failure %d\n", error);
488 /* Set admin queue parameters */
489 hw->aq.num_arq_entries = IXL_AQ_LEN;
490 hw->aq.num_asq_entries = IXL_AQ_LEN;
491 hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
492 hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
494 /* Initialize mac filter list for VSI */
495 SLIST_INIT(&vsi->ftl);
497 /* Initialize the shared code */
498 error = i40e_init_shared_code(hw);
500 device_printf(dev, "Unable to initialize shared code, error %d\n",
506 /* Set up the admin queue */
507 error = i40e_init_adminq(hw);
508 if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
509 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
514 ixl_print_nvm_version(pf);
516 if (error == I40E_ERR_FIRMWARE_API_VERSION) {
517 device_printf(dev, "The driver for the device stopped "
518 "because the NVM image is newer than expected.\n"
519 "You must install the most recent version of "
520 "the network driver.\n");
525 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
526 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
527 device_printf(dev, "The driver for the device detected "
528 "a newer version of the NVM image than expected.\n"
529 "Please install the most recent version of the network driver.\n");
530 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
531 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
532 device_printf(dev, "The driver for the device detected "
533 "an older version of the NVM image than expected.\n"
534 "Please update the NVM image.\n");
537 i40e_clear_pxe_mode(hw);
539 /* Get capabilities from the device */
540 error = ixl_get_hw_capabilities(pf);
542 device_printf(dev, "HW capabilities failure!\n");
546 /* Set up host memory cache */
547 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
548 hw->func_caps.num_rx_qp, 0, 0);
550 device_printf(dev, "init_lan_hmc failed: %d\n", error);
554 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
556 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
560 /* Disable LLDP from the firmware for certain NVM versions */
561 if (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
562 (pf->hw.aq.fw_maj_ver < 4))
563 i40e_aq_stop_lldp(hw, TRUE, NULL);
565 i40e_get_mac_addr(hw, hw->mac.addr);
566 error = i40e_validate_mac_addr(hw->mac.addr);
568 device_printf(dev, "validate_mac_addr failed: %d\n", error);
571 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
572 i40e_get_port_mac_addr(hw, hw->mac.port_addr);
574 /* Set up VSI and queues */
575 if (ixl_setup_stations(pf) != 0) {
576 device_printf(dev, "setup stations failed!\n");
581 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
582 (hw->aq.fw_maj_ver < 4)) {
584 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
586 device_printf(dev, "link restart failed, aq_err=%d\n",
587 pf->hw.aq.asq_last_status);
592 /* Determine link state */
593 hw->phy.get_link_info = TRUE;
594 i40e_get_link_status(hw, &pf->link_up);
596 /* Setup OS network interface / ifnet */
597 if (ixl_setup_interface(dev, vsi) != 0) {
598 device_printf(dev, "interface setup failed!\n");
603 error = ixl_switch_config(pf);
605 device_printf(dev, "Initial ixl_switch_config() failed: %d\n",
610 /* Limit PHY interrupts to link, autoneg, and modules failure */
611 error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
614 device_printf(dev, "i40e_aq_set_phy_mask() failed: err %d,"
615 " aq_err %d\n", error, hw->aq.asq_last_status);
619 /* Get the bus configuration and set the shared code's config */
620 bus = ixl_get_bus_info(hw, dev);
621 i40e_set_pci_config_data(hw, bus);
624 * In MSI-X mode, initialize the Admin Queue interrupt,
625 * so userland tools can communicate with the adapter regardless of
626 * the ifnet interface's status.
629 error = ixl_setup_adminq_msix(pf);
631 device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
635 error = ixl_setup_adminq_tq(pf);
637 device_printf(dev, "ixl_setup_adminq_tq error: %d\n",
641 ixl_configure_intr0_msix(pf);
642 ixl_enable_adminq(hw);
645 /* Initialize statistics & add sysctls */
646 ixl_add_device_sysctls(pf);
648 ixl_pf_reset_stats(pf);
649 ixl_update_stats_counters(pf);
650 ixl_add_hw_stats(pf);
652 /* Register for VLAN events */
653 vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
654 ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
655 vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
656 ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
659 /* SR-IOV is only supported when MSI-X is in use. */
661 pf_schema = pci_iov_schema_alloc_node();
662 vf_schema = pci_iov_schema_alloc_node();
663 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
664 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
665 IOV_SCHEMA_HASDEFAULT, TRUE);
666 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
667 IOV_SCHEMA_HASDEFAULT, FALSE);
668 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
669 IOV_SCHEMA_HASDEFAULT, FALSE);
671 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
672 if (iov_error != 0) {
674 "Failed to initialize SR-IOV (error=%d)\n",
677 device_printf(dev, "SR-IOV ready\n");
682 ixl_netmap_attach(vsi);
683 #endif /* DEV_NETMAP */
684 INIT_DEBUGOUT("ixl_attach: end");
688 if (vsi->ifp != NULL)
691 i40e_shutdown_lan_hmc(hw);
693 i40e_shutdown_adminq(hw);
695 ixl_free_pci_resources(pf);
697 IXL_PF_LOCK_DESTROY(pf);
701 /*********************************************************************
702 * Device removal routine
704 * The detach entry point is called when the driver is being removed.
705 * This routine stops the adapter and deallocates all the resources
706 * that were allocated for driver operation.
708 * return 0 on success, positive on failure
709 *********************************************************************/
712 ixl_detach(device_t dev)
714 struct ixl_pf *pf = device_get_softc(dev);
715 struct i40e_hw *hw = &pf->hw;
716 struct ixl_vsi *vsi = &pf->vsi;
717 enum i40e_status_code status;
722 INIT_DEBUGOUT("ixl_detach: begin");
724 /* Make sure VLANS are not using driver */
725 if (vsi->ifp->if_vlantrunk != NULL) {
726 device_printf(dev, "Vlan in use, detach first\n");
731 error = pci_iov_detach(dev);
733 device_printf(dev, "SR-IOV in use; detach first.\n");
738 ether_ifdetach(vsi->ifp);
739 if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)
742 ixl_free_queue_tqs(vsi);
744 /* Shutdown LAN HMC */
745 status = i40e_shutdown_lan_hmc(hw);
748 "Shutdown LAN HMC failed with code %d\n", status);
750 /* Shutdown admin queue */
751 ixl_disable_adminq(hw);
752 ixl_free_adminq_tq(pf);
753 ixl_teardown_adminq_msix(pf);
754 status = i40e_shutdown_adminq(hw);
757 "Shutdown Admin queue failed with code %d\n", status);
759 /* Unregister VLAN events */
760 if (vsi->vlan_attach != NULL)
761 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
762 if (vsi->vlan_detach != NULL)
763 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
765 callout_drain(&pf->timer);
767 netmap_detach(vsi->ifp);
768 #endif /* DEV_NETMAP */
769 ixl_free_pci_resources(pf);
770 bus_generic_detach(dev);
773 IXL_PF_LOCK_DESTROY(pf);
777 /*********************************************************************
779 * Shutdown entry point
781 **********************************************************************/
784 ixl_shutdown(device_t dev)
786 struct ixl_pf *pf = device_get_softc(dev);
792 /*********************************************************************
794 * Get the hardware capabilities
796 **********************************************************************/
799 ixl_get_hw_capabilities(struct ixl_pf *pf)
801 struct i40e_aqc_list_capabilities_element_resp *buf;
802 struct i40e_hw *hw = &pf->hw;
803 device_t dev = pf->dev;
808 len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
810 if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
811 malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
812 device_printf(dev, "Unable to allocate cap memory\n");
816 /* This populates the hw struct */
817 error = i40e_aq_discover_capabilities(hw, buf, len,
818 &needed, i40e_aqc_opc_list_func_capabilities, NULL);
820 if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
822 /* retry once with a larger buffer */
826 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
827 device_printf(dev, "capability discovery failed: %d\n",
828 pf->hw.aq.asq_last_status);
832 /* Capture this PF's starting queue pair */
833 pf->qbase = hw->func_caps.base_queue;
836 device_printf(dev, "pf_id=%d, num_vfs=%d, msix_pf=%d, "
837 "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
838 hw->pf_id, hw->func_caps.num_vfs,
839 hw->func_caps.num_msix_vectors,
840 hw->func_caps.num_msix_vectors_vf,
841 hw->func_caps.fd_filters_guaranteed,
842 hw->func_caps.fd_filters_best_effort,
843 hw->func_caps.num_tx_qp,
844 hw->func_caps.num_rx_qp,
845 hw->func_caps.base_queue);
851 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
853 device_t dev = vsi->dev;
855 /* Enable/disable TXCSUM/TSO4 */
856 if (!(ifp->if_capenable & IFCAP_TXCSUM)
857 && !(ifp->if_capenable & IFCAP_TSO4)) {
858 if (mask & IFCAP_TXCSUM) {
859 ifp->if_capenable |= IFCAP_TXCSUM;
860 /* enable TXCSUM, restore TSO if previously enabled */
861 if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
862 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
863 ifp->if_capenable |= IFCAP_TSO4;
866 else if (mask & IFCAP_TSO4) {
867 ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
868 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
870 "TSO4 requires txcsum, enabling both...\n");
872 } else if((ifp->if_capenable & IFCAP_TXCSUM)
873 && !(ifp->if_capenable & IFCAP_TSO4)) {
874 if (mask & IFCAP_TXCSUM)
875 ifp->if_capenable &= ~IFCAP_TXCSUM;
876 else if (mask & IFCAP_TSO4)
877 ifp->if_capenable |= IFCAP_TSO4;
878 } else if((ifp->if_capenable & IFCAP_TXCSUM)
879 && (ifp->if_capenable & IFCAP_TSO4)) {
880 if (mask & IFCAP_TXCSUM) {
881 vsi->flags |= IXL_FLAGS_KEEP_TSO4;
882 ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
884 "TSO4 requires txcsum, disabling both...\n");
885 } else if (mask & IFCAP_TSO4)
886 ifp->if_capenable &= ~IFCAP_TSO4;
889 /* Enable/disable TXCSUM_IPV6/TSO6 */
890 if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
891 && !(ifp->if_capenable & IFCAP_TSO6)) {
892 if (mask & IFCAP_TXCSUM_IPV6) {
893 ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
894 if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
895 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
896 ifp->if_capenable |= IFCAP_TSO6;
898 } else if (mask & IFCAP_TSO6) {
899 ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
900 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
902 "TSO6 requires txcsum6, enabling both...\n");
904 } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
905 && !(ifp->if_capenable & IFCAP_TSO6)) {
906 if (mask & IFCAP_TXCSUM_IPV6)
907 ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
908 else if (mask & IFCAP_TSO6)
909 ifp->if_capenable |= IFCAP_TSO6;
910 } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
911 && (ifp->if_capenable & IFCAP_TSO6)) {
912 if (mask & IFCAP_TXCSUM_IPV6) {
913 vsi->flags |= IXL_FLAGS_KEEP_TSO6;
914 ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
916 "TSO6 requires txcsum6, disabling both...\n");
917 } else if (mask & IFCAP_TSO6)
918 ifp->if_capenable &= ~IFCAP_TSO6;
922 /*********************************************************************
925 * ixl_ioctl is called when the user wants to configure the
928 * return 0 on success, positive on failure
929 **********************************************************************/
932 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
934 struct ixl_vsi *vsi = ifp->if_softc;
935 struct ixl_pf *pf = vsi->back;
936 struct ifreq *ifr = (struct ifreq *)data;
937 struct ifdrv *ifd = (struct ifdrv *)data;
938 #if defined(INET) || defined(INET6)
939 struct ifaddr *ifa = (struct ifaddr *)data;
940 bool avoid_reset = FALSE;
948 if (ifa->ifa_addr->sa_family == AF_INET)
952 if (ifa->ifa_addr->sa_family == AF_INET6)
955 #if defined(INET) || defined(INET6)
957 ** Calling init results in link renegotiation,
958 ** so we avoid doing it when possible.
961 ifp->if_flags |= IFF_UP;
962 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
965 if (!(ifp->if_flags & IFF_NOARP))
966 arp_ifinit(ifp, ifa);
969 error = ether_ioctl(ifp, command, data);
973 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
974 if (ifr->ifr_mtu > IXL_MAX_FRAME -
975 ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
979 ifp->if_mtu = ifr->ifr_mtu;
980 vsi->max_frame_size =
981 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
982 + ETHER_VLAN_ENCAP_LEN;
983 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
989 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
991 if (ifp->if_flags & IFF_UP) {
992 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
993 if ((ifp->if_flags ^ pf->if_flags) &
994 (IFF_PROMISC | IFF_ALLMULTI)) {
995 ixl_set_promisc(vsi);
1003 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1009 pf->if_flags = ifp->if_flags;
1014 IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific "
1017 /* NVM update command */
1018 if (ifd->ifd_cmd == I40E_NVM_ACCESS)
1019 error = ixl_handle_nvmupd_cmd(pf, ifd);
1024 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1025 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1027 ixl_disable_intr(vsi);
1029 ixl_enable_intr(vsi);
1034 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1035 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1037 ixl_disable_intr(vsi);
1039 ixl_enable_intr(vsi);
1045 #ifdef IFM_ETH_XTYPE
1048 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1049 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1053 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1054 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1056 ixl_cap_txcsum_tso(vsi, ifp, mask);
1058 if (mask & IFCAP_RXCSUM)
1059 ifp->if_capenable ^= IFCAP_RXCSUM;
1060 if (mask & IFCAP_RXCSUM_IPV6)
1061 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1062 if (mask & IFCAP_LRO)
1063 ifp->if_capenable ^= IFCAP_LRO;
1064 if (mask & IFCAP_VLAN_HWTAGGING)
1065 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1066 if (mask & IFCAP_VLAN_HWFILTER)
1067 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1068 if (mask & IFCAP_VLAN_HWTSO)
1069 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1070 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1072 ixl_init_locked(pf);
1075 VLAN_CAPABILITIES(ifp);
1081 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1082 error = ether_ioctl(ifp, command, data);
1090 /*********************************************************************
1093 * This routine is used in two ways. It is used by the stack as
1094 * init entry point in network interface structure. It is also used
1095 * by the driver as a hw/sw initialization routine to get to a
1098 * return 0 on success, positive on failure
1099 **********************************************************************/
1102 ixl_init_locked(struct ixl_pf *pf)
1104 struct i40e_hw *hw = &pf->hw;
1105 struct ixl_vsi *vsi = &pf->vsi;
1106 struct ifnet *ifp = vsi->ifp;
1107 device_t dev = pf->dev;
1108 struct i40e_filter_control_settings filter;
1109 u8 tmpaddr[ETHER_ADDR_LEN];
1112 mtx_assert(&pf->pf_mtx, MA_OWNED);
1113 INIT_DEBUGOUT("ixl_init_locked: begin");
1115 ixl_stop_locked(pf);
1117 /* Get the latest mac address... User might use a LAA */
1118 bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1119 I40E_ETH_LENGTH_OF_ADDRESS);
1120 if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1121 (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
1122 ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1123 bcopy(tmpaddr, hw->mac.addr,
1124 I40E_ETH_LENGTH_OF_ADDRESS);
1125 ret = i40e_aq_mac_address_write(hw,
1126 I40E_AQC_WRITE_TYPE_LAA_ONLY,
1127 hw->mac.addr, NULL);
1129 device_printf(dev, "LLA address"
1130 "change failed!!\n");
1135 ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1137 /* Set the various hardware offload abilities */
1138 ifp->if_hwassist = 0;
1139 if (ifp->if_capenable & IFCAP_TSO)
1140 ifp->if_hwassist |= CSUM_TSO;
1141 if (ifp->if_capenable & IFCAP_TXCSUM)
1142 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1143 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1144 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1146 /* Set up the device filtering */
1147 bzero(&filter, sizeof(filter));
1148 filter.enable_ethtype = TRUE;
1149 filter.enable_macvlan = TRUE;
1151 filter.enable_fdir = TRUE;
1153 filter.hash_lut_size = I40E_HASH_LUT_SIZE_512;
1154 if (i40e_set_filter_control(hw, &filter))
1155 device_printf(dev, "i40e_set_filter_control() failed\n");
1158 ixl_config_rss(vsi);
1160 /* Prepare the VSI: rings, hmc contexts, etc... */
1161 if (ixl_initialize_vsi(vsi)) {
1162 device_printf(dev, "initialize vsi failed!!\n");
1166 /* Add protocol filters to list */
1167 ixl_init_filters(vsi);
1169 /* Setup vlan's if needed */
1170 ixl_setup_vlan_filters(vsi);
1172 /* Set up MSI/X routing and the ITR settings */
1173 if (ixl_enable_msix) {
1174 ixl_configure_queue_intr_msix(pf);
1175 ixl_configure_itr(pf);
1177 ixl_configure_legacy(pf);
1179 ixl_enable_rings(vsi);
1181 i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1183 ixl_reconfigure_filters(vsi);
1185 /* And now turn on interrupts */
1186 ixl_enable_intr(vsi);
1189 hw->phy.get_link_info = TRUE;
1190 i40e_get_link_status(hw, &pf->link_up);
1191 ixl_update_link_status(pf);
1193 /* Set initial advertised speed sysctl value */
1194 ixl_get_initial_advertised_speeds(pf);
1196 /* Start the local timer */
1197 callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1199 /* Now inform the stack we're ready */
1200 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1205 /* For the set_advertise sysctl */
1207 ixl_get_initial_advertised_speeds(struct ixl_pf *pf)
1209 struct i40e_hw *hw = &pf->hw;
1210 device_t dev = pf->dev;
1211 enum i40e_status_code status;
1212 struct i40e_aq_get_phy_abilities_resp abilities;
1214 /* Set initial sysctl values */
1215 status = i40e_aq_get_phy_capabilities(hw, FALSE, false, &abilities,
1218 /* Non-fatal error */
1219 device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n",
1224 if (abilities.link_speed & I40E_LINK_SPEED_40GB)
1225 pf->advertised_speed |= 0x10;
1226 if (abilities.link_speed & I40E_LINK_SPEED_20GB)
1227 pf->advertised_speed |= 0x8;
1228 if (abilities.link_speed & I40E_LINK_SPEED_10GB)
1229 pf->advertised_speed |= 0x4;
1230 if (abilities.link_speed & I40E_LINK_SPEED_1GB)
1231 pf->advertised_speed |= 0x2;
1232 if (abilities.link_speed & I40E_LINK_SPEED_100MB)
1233 pf->advertised_speed |= 0x1;
1237 ixl_teardown_hw_structs(struct ixl_pf *pf)
1239 enum i40e_status_code status = 0;
1240 struct i40e_hw *hw = &pf->hw;
1241 device_t dev = pf->dev;
1243 /* Shutdown LAN HMC */
1244 if (hw->hmc.hmc_obj) {
1245 status = i40e_shutdown_lan_hmc(hw);
1248 "init: LAN HMC shutdown failure; status %d\n", status);
1253 // XXX: This gets called when we know the adminq is inactive;
1254 // so we already know it's setup when we get here.
1256 /* Shutdown admin queue */
1257 status = i40e_shutdown_adminq(hw);
1260 "init: Admin Queue shutdown failure; status %d\n", status);
1267 ixl_reset(struct ixl_pf *pf)
1269 struct i40e_hw *hw = &pf->hw;
1270 device_t dev = pf->dev;
1274 // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary
1276 error = i40e_pf_reset(hw);
1278 device_printf(dev, "init: PF reset failure");
1283 error = i40e_init_adminq(hw);
1285 device_printf(dev, "init: Admin queue init failure;"
1286 " status code %d", error);
1291 i40e_clear_pxe_mode(hw);
1293 error = ixl_get_hw_capabilities(pf);
1295 device_printf(dev, "init: Error retrieving HW capabilities;"
1296 " status code %d\n", error);
1300 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1301 hw->func_caps.num_rx_qp, 0, 0);
1303 device_printf(dev, "init: LAN HMC init failed; status code %d\n",
1309 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1311 device_printf(dev, "init: LAN HMC config failed; status code %d\n",
1317 // XXX: possible fix for panic, but our failure recovery is still broken
1318 error = ixl_switch_config(pf);
1320 device_printf(dev, "init: ixl_switch_config() failed: %d\n",
1325 error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
1328 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
1329 " aq_err %d\n", error, hw->aq.asq_last_status);
1334 error = i40e_set_fc(hw, &set_fc_err_mask, true);
1336 device_printf(dev, "init: setting link flow control failed; retcode %d,"
1337 " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
1341 // XXX: (Rebuild VSIs?)
1343 /* Firmware delay workaround */
1344 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
1345 (hw->aq.fw_maj_ver < 4)) {
1346 i40e_msec_delay(75);
1347 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
1349 device_printf(dev, "init: link restart failed, aq_err %d\n",
1350 hw->aq.asq_last_status);
1363 struct ixl_pf *pf = arg;
1364 struct ixl_vsi *vsi = &pf->vsi;
1365 device_t dev = pf->dev;
1369 * If the aq is dead here, it probably means something outside of the driver
1370 * did something to the adapter, like a PF reset.
1371 * So rebuild the driver's state here if that occurs.
1373 if (!i40e_check_asq_alive(&pf->hw)) {
1374 device_printf(dev, "Admin Queue is down; resetting...\n");
1376 ixl_teardown_hw_structs(pf);
1382 * Set up LAN queue interrupts here.
1383 * Kernel interrupt setup functions cannot be called while holding a lock,
1384 * so this is done outside of init_locked().
1387 /* Teardown existing interrupts, if they exist */
1388 ixl_teardown_queue_msix(vsi);
1389 ixl_free_queue_tqs(vsi);
1390 /* Then set them up again */
1391 error = ixl_setup_queue_msix(vsi);
1393 device_printf(dev, "ixl_setup_queue_msix() error: %d\n",
1395 error = ixl_setup_queue_tqs(vsi);
1397 device_printf(dev, "ixl_setup_queue_tqs() error: %d\n",
1401 error = ixl_assign_vsi_legacy(pf);
1403 device_printf(pf->dev, "assign_vsi_msix/legacy error: %d\n", error);
1408 ixl_init_locked(pf);
1413 ** MSIX Interrupt Handlers and Tasklets
1416 ixl_handle_que(void *context, int pending)
1418 struct ixl_queue *que = context;
1419 struct ixl_vsi *vsi = que->vsi;
1420 struct i40e_hw *hw = vsi->hw;
1421 struct tx_ring *txr = &que->txr;
1422 struct ifnet *ifp = vsi->ifp;
1425 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1426 more = ixl_rxeof(que, IXL_RX_LIMIT);
1429 if (!drbr_empty(ifp, txr->br))
1430 ixl_mq_start_locked(ifp, txr);
1433 taskqueue_enqueue(que->tq, &que->task);
1438 /* Reenable this interrupt - hmmm */
1439 ixl_enable_queue(hw, que->me);
1444 /*********************************************************************
1446 * Legacy Interrupt Service routine
1448 **********************************************************************/
1452 struct ixl_pf *pf = arg;
1453 struct i40e_hw *hw = &pf->hw;
1454 struct ixl_vsi *vsi = &pf->vsi;
1455 struct ixl_queue *que = vsi->queues;
1456 struct ifnet *ifp = vsi->ifp;
1457 struct tx_ring *txr = &que->txr;
1458 u32 reg, icr0, mask;
1459 bool more_tx, more_rx;
1463 /* Protect against spurious interrupts */
1464 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1467 icr0 = rd32(hw, I40E_PFINT_ICR0);
1469 reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1470 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1471 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1473 mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1476 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1477 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1480 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1481 taskqueue_enqueue(pf->tq, &pf->adminq);
1485 more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1488 more_tx = ixl_txeof(que);
1489 if (!drbr_empty(vsi->ifp, txr->br))
1493 /* re-enable other interrupt causes */
1494 wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1496 /* And now the queues */
1497 reg = rd32(hw, I40E_QINT_RQCTL(0));
1498 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1499 wr32(hw, I40E_QINT_RQCTL(0), reg);
1501 reg = rd32(hw, I40E_QINT_TQCTL(0));
1502 reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1503 reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1504 wr32(hw, I40E_QINT_TQCTL(0), reg);
1506 ixl_enable_legacy(hw);
1512 /*********************************************************************
1514 * MSIX VSI Interrupt Service routine
1516 **********************************************************************/
1518 ixl_msix_que(void *arg)
1520 struct ixl_queue *que = arg;
1521 struct ixl_vsi *vsi = que->vsi;
1522 struct i40e_hw *hw = vsi->hw;
1523 struct tx_ring *txr = &que->txr;
1524 bool more_tx, more_rx;
1526 /* Protect against spurious interrupts */
1527 if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1532 more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1535 more_tx = ixl_txeof(que);
1537 ** Make certain that if the stack
1538 ** has anything queued the task gets
1539 ** scheduled to handle it.
1541 if (!drbr_empty(vsi->ifp, txr->br))
1545 ixl_set_queue_rx_itr(que);
1546 ixl_set_queue_tx_itr(que);
1548 if (more_tx || more_rx)
1549 taskqueue_enqueue(que->tq, &que->task);
1551 ixl_enable_queue(hw, que->me);
1557 /*********************************************************************
1559 * MSIX Admin Queue Interrupt Service routine
1561 **********************************************************************/
1563 ixl_msix_adminq(void *arg)
1565 struct ixl_pf *pf = arg;
1566 struct i40e_hw *hw = &pf->hw;
1567 u32 reg, mask, rstat_reg;
1568 bool do_task = FALSE;
1572 reg = rd32(hw, I40E_PFINT_ICR0);
1573 mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1575 /* Check on the cause */
1576 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) {
1577 mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK;
1581 if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1582 ixl_handle_mdd_event(pf);
1583 mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK;
1586 if (reg & I40E_PFINT_ICR0_GRST_MASK) {
1587 device_printf(pf->dev, "Reset Requested!\n");
1588 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT);
1589 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
1590 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
1591 device_printf(pf->dev, "Reset type: ");
1592 switch (rstat_reg) {
1593 /* These others might be handled similarly to an EMPR reset */
1594 case I40E_RESET_CORER:
1597 case I40E_RESET_GLOBR:
1600 case I40E_RESET_EMPR:
1602 atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
1608 // overload admin queue task to check reset progress?
1612 if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) {
1613 device_printf(pf->dev, "ECC Error detected!\n");
1616 if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) {
1617 device_printf(pf->dev, "HMC Error detected!\n");
1620 if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) {
1621 device_printf(pf->dev, "PCI Exception detected!\n");
1625 if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1626 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1627 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1631 reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1632 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1633 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1636 taskqueue_enqueue(pf->tq, &pf->adminq);
1639 /*********************************************************************
1641 * Media Ioctl callback
1643 * This routine is called whenever the user queries the status of
1644 * the interface using ifconfig.
1646 **********************************************************************/
1648 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1650 struct ixl_vsi *vsi = ifp->if_softc;
1651 struct ixl_pf *pf = vsi->back;
1652 struct i40e_hw *hw = &pf->hw;
1654 INIT_DEBUGOUT("ixl_media_status: begin");
1657 hw->phy.get_link_info = TRUE;
1658 i40e_get_link_status(hw, &pf->link_up);
1659 ixl_update_link_status(pf);
1661 ifmr->ifm_status = IFM_AVALID;
1662 ifmr->ifm_active = IFM_ETHER;
1669 ifmr->ifm_status |= IFM_ACTIVE;
1671 /* Hardware always does full-duplex */
1672 ifmr->ifm_active |= IFM_FDX;
1674 switch (hw->phy.link_info.phy_type) {
1676 case I40E_PHY_TYPE_100BASE_TX:
1677 ifmr->ifm_active |= IFM_100_TX;
1680 case I40E_PHY_TYPE_1000BASE_T:
1681 ifmr->ifm_active |= IFM_1000_T;
1683 case I40E_PHY_TYPE_1000BASE_SX:
1684 ifmr->ifm_active |= IFM_1000_SX;
1686 case I40E_PHY_TYPE_1000BASE_LX:
1687 ifmr->ifm_active |= IFM_1000_LX;
1689 case I40E_PHY_TYPE_1000BASE_T_OPTICAL:
1690 ifmr->ifm_active |= IFM_OTHER;
1693 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1694 ifmr->ifm_active |= IFM_10G_TWINAX;
1696 case I40E_PHY_TYPE_10GBASE_SR:
1697 ifmr->ifm_active |= IFM_10G_SR;
1699 case I40E_PHY_TYPE_10GBASE_LR:
1700 ifmr->ifm_active |= IFM_10G_LR;
1702 case I40E_PHY_TYPE_10GBASE_T:
1703 ifmr->ifm_active |= IFM_10G_T;
1705 case I40E_PHY_TYPE_XAUI:
1706 case I40E_PHY_TYPE_XFI:
1707 case I40E_PHY_TYPE_10GBASE_AOC:
1708 ifmr->ifm_active |= IFM_OTHER;
1711 case I40E_PHY_TYPE_40GBASE_CR4:
1712 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1713 ifmr->ifm_active |= IFM_40G_CR4;
1715 case I40E_PHY_TYPE_40GBASE_SR4:
1716 ifmr->ifm_active |= IFM_40G_SR4;
1718 case I40E_PHY_TYPE_40GBASE_LR4:
1719 ifmr->ifm_active |= IFM_40G_LR4;
1721 case I40E_PHY_TYPE_XLAUI:
1722 ifmr->ifm_active |= IFM_OTHER;
1724 #ifndef IFM_ETH_XTYPE
1725 case I40E_PHY_TYPE_1000BASE_KX:
1726 ifmr->ifm_active |= IFM_1000_CX;
1728 case I40E_PHY_TYPE_SGMII:
1729 ifmr->ifm_active |= IFM_OTHER;
1731 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1732 case I40E_PHY_TYPE_10GBASE_CR1:
1733 ifmr->ifm_active |= IFM_10G_TWINAX;
1735 case I40E_PHY_TYPE_10GBASE_KX4:
1736 ifmr->ifm_active |= IFM_10G_CX4;
1738 case I40E_PHY_TYPE_10GBASE_KR:
1739 ifmr->ifm_active |= IFM_10G_SR;
1741 case I40E_PHY_TYPE_SFI:
1742 ifmr->ifm_active |= IFM_OTHER;
1744 case I40E_PHY_TYPE_40GBASE_KR4:
1745 case I40E_PHY_TYPE_XLPPI:
1746 case I40E_PHY_TYPE_40GBASE_AOC:
1747 ifmr->ifm_active |= IFM_40G_SR4;
1750 case I40E_PHY_TYPE_1000BASE_KX:
1751 ifmr->ifm_active |= IFM_1000_KX;
1753 case I40E_PHY_TYPE_SGMII:
1754 ifmr->ifm_active |= IFM_1000_SGMII;
1756 /* ERJ: What's the difference between these? */
1757 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1758 case I40E_PHY_TYPE_10GBASE_CR1:
1759 ifmr->ifm_active |= IFM_10G_CR1;
1761 case I40E_PHY_TYPE_10GBASE_KX4:
1762 ifmr->ifm_active |= IFM_10G_KX4;
1764 case I40E_PHY_TYPE_10GBASE_KR:
1765 ifmr->ifm_active |= IFM_10G_KR;
1767 case I40E_PHY_TYPE_SFI:
1768 ifmr->ifm_active |= IFM_10G_SFI;
1770 /* Our single 20G media type */
1771 case I40E_PHY_TYPE_20GBASE_KR2:
1772 ifmr->ifm_active |= IFM_20G_KR2;
1774 case I40E_PHY_TYPE_40GBASE_KR4:
1775 ifmr->ifm_active |= IFM_40G_KR4;
1777 case I40E_PHY_TYPE_XLPPI:
1778 case I40E_PHY_TYPE_40GBASE_AOC:
1779 ifmr->ifm_active |= IFM_40G_XLPPI;
1782 /* Unknown to driver */
1784 ifmr->ifm_active |= IFM_UNKNOWN;
1787 /* Report flow control status as well */
1788 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1789 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1790 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1791 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1799 * NOTE: Fortville does not support forcing media speeds. Instead,
1800 * use the set_advertise sysctl to set the speeds Fortville
1801 * will advertise or be allowed to operate at.
1804 ixl_media_change(struct ifnet * ifp)
1806 struct ixl_vsi *vsi = ifp->if_softc;
1807 struct ifmedia *ifm = &vsi->media;
1809 INIT_DEBUGOUT("ixl_media_change: begin");
1811 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1814 if_printf(ifp, "Media change is not supported.\n");
1822 ** ATR: Application Targetted Receive - creates a filter
1823 ** based on TX flow info that will keep the receive
1824 ** portion of the flow on the same queue. Based on the
1825 ** implementation this is only available for TCP connections
1828 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1830 struct ixl_vsi *vsi = que->vsi;
1831 struct tx_ring *txr = &que->txr;
1832 struct i40e_filter_program_desc *FDIR;
1836 /* check if ATR is enabled and sample rate */
1837 if ((!ixl_enable_fdir) || (!txr->atr_rate))
1840 ** We sample all TCP SYN/FIN packets,
1841 ** or at the selected sample rate
1844 if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1845 (txr->atr_count < txr->atr_rate))
1849 /* Get a descriptor to use */
1850 idx = txr->next_avail;
1851 FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1852 if (++idx == que->num_desc)
1855 txr->next_avail = idx;
1857 ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1858 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1860 ptype |= (etype == ETHERTYPE_IP) ?
1861 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1862 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1863 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1864 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1866 ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1868 dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1871 ** We use the TCP TH_FIN as a trigger to remove
1872 ** the filter, otherwise its an update.
1874 dtype |= (th->th_flags & TH_FIN) ?
1875 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1876 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1877 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1878 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1880 dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1881 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1883 dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1884 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1886 FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1887 FDIR->dtype_cmd_cntindex = htole32(dtype);
1894 ixl_set_promisc(struct ixl_vsi *vsi)
1896 struct ifnet *ifp = vsi->ifp;
1897 struct i40e_hw *hw = vsi->hw;
1899 bool uni = FALSE, multi = FALSE;
1901 if (ifp->if_flags & IFF_ALLMULTI)
1903 else { /* Need to count the multicast addresses */
1904 struct ifmultiaddr *ifma;
1905 if_maddr_rlock(ifp);
1906 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1907 if (ifma->ifma_addr->sa_family != AF_LINK)
1909 if (mcnt == MAX_MULTICAST_ADDR)
1913 if_maddr_runlock(ifp);
1916 if (mcnt >= MAX_MULTICAST_ADDR)
1918 if (ifp->if_flags & IFF_PROMISC)
1921 err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1922 vsi->seid, uni, NULL);
1923 err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1924 vsi->seid, multi, NULL);
1928 /*********************************************************************
1931 * Routines for multicast and vlan filter management.
1933 *********************************************************************/
1935 ixl_add_multi(struct ixl_vsi *vsi)
1937 struct ifmultiaddr *ifma;
1938 struct ifnet *ifp = vsi->ifp;
1939 struct i40e_hw *hw = vsi->hw;
1940 int mcnt = 0, flags;
1942 IOCTL_DEBUGOUT("ixl_add_multi: begin");
1944 if_maddr_rlock(ifp);
1946 ** First just get a count, to decide if we
1947 ** we simply use multicast promiscuous.
1949 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1950 if (ifma->ifma_addr->sa_family != AF_LINK)
1954 if_maddr_runlock(ifp);
1956 if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1957 /* delete existing MC filters */
1958 ixl_del_hw_filters(vsi, mcnt);
1959 i40e_aq_set_vsi_multicast_promiscuous(hw,
1960 vsi->seid, TRUE, NULL);
1965 if_maddr_rlock(ifp);
1966 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1967 if (ifma->ifma_addr->sa_family != AF_LINK)
1969 ixl_add_mc_filter(vsi,
1970 (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1973 if_maddr_runlock(ifp);
1975 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1976 ixl_add_hw_filters(vsi, flags, mcnt);
1979 IOCTL_DEBUGOUT("ixl_add_multi: end");
1984 ixl_del_multi(struct ixl_vsi *vsi)
1986 struct ifnet *ifp = vsi->ifp;
1987 struct ifmultiaddr *ifma;
1988 struct ixl_mac_filter *f;
1992 IOCTL_DEBUGOUT("ixl_del_multi: begin");
1994 /* Search for removed multicast addresses */
1995 if_maddr_rlock(ifp);
1996 SLIST_FOREACH(f, &vsi->ftl, next) {
1997 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1999 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2000 if (ifma->ifma_addr->sa_family != AF_LINK)
2002 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2003 if (cmp_etheraddr(f->macaddr, mc_addr)) {
2008 if (match == FALSE) {
2009 f->flags |= IXL_FILTER_DEL;
2014 if_maddr_runlock(ifp);
2017 ixl_del_hw_filters(vsi, mcnt);
2021 /*********************************************************************
2024 * This routine checks for link status,updates statistics,
2025 * and runs the watchdog check.
2027 * Only runs when the driver is configured UP and RUNNING.
2029 **********************************************************************/
2032 ixl_local_timer(void *arg)
2034 struct ixl_pf *pf = arg;
2035 struct i40e_hw *hw = &pf->hw;
2036 struct ixl_vsi *vsi = &pf->vsi;
2037 struct ixl_queue *que = vsi->queues;
2038 device_t dev = pf->dev;
2042 mtx_assert(&pf->pf_mtx, MA_OWNED);
2044 /* Fire off the adminq task */
2045 taskqueue_enqueue(pf->tq, &pf->adminq);
2048 ixl_update_stats_counters(pf);
2051 ** Check status of the queues
2053 mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
2054 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
2056 for (int i = 0; i < vsi->num_queues; i++, que++) {
2057 /* Any queues with outstanding work get a sw irq */
2059 wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
2061 ** Each time txeof runs without cleaning, but there
2062 ** are uncleaned descriptors it increments busy. If
2063 ** we get to 5 we declare it hung.
2065 if (que->busy == IXL_QUEUE_HUNG) {
2067 /* Mark the queue as inactive */
2068 vsi->active_queues &= ~((u64)1 << que->me);
2071 /* Check if we've come back from hung */
2072 if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
2073 vsi->active_queues |= ((u64)1 << que->me);
2075 if (que->busy >= IXL_MAX_TX_BUSY) {
2077 device_printf(dev,"Warning queue %d "
2078 "appears to be hung!\n", i);
2080 que->busy = IXL_QUEUE_HUNG;
2084 /* Only reinit if all queues show hung */
2085 if (hung == vsi->num_queues)
2088 callout_reset(&pf->timer, hz, ixl_local_timer, pf);
2092 device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
2093 ixl_init_locked(pf);
2097 ** Note: this routine updates the OS on the link state
2098 ** the real check of the hardware only happens with
2099 ** a link interrupt.
2102 ixl_update_link_status(struct ixl_pf *pf)
2104 struct ixl_vsi *vsi = &pf->vsi;
2105 struct i40e_hw *hw = &pf->hw;
2106 struct ifnet *ifp = vsi->ifp;
2107 device_t dev = pf->dev;
2110 if (vsi->link_active == FALSE) {
2111 pf->fc = hw->fc.current_mode;
2113 device_printf(dev,"Link is up %d Gbps %s,"
2114 " Flow Control: %s\n",
2116 I40E_LINK_SPEED_40GB)? 40:10),
2117 "Full Duplex", ixl_fc_string[pf->fc]);
2119 vsi->link_active = TRUE;
2121 ** Warn user if link speed on NPAR enabled
2122 ** partition is not at least 10GB
2124 if (hw->func_caps.npar_enable &&
2125 (hw->phy.link_info.link_speed ==
2126 I40E_LINK_SPEED_1GB ||
2127 hw->phy.link_info.link_speed ==
2128 I40E_LINK_SPEED_100MB))
2129 device_printf(dev, "The partition detected"
2130 "link speed that is less than 10Gbps\n");
2131 if_link_state_change(ifp, LINK_STATE_UP);
2133 } else { /* Link down */
2134 if (vsi->link_active == TRUE) {
2136 device_printf(dev, "Link is Down\n");
2137 if_link_state_change(ifp, LINK_STATE_DOWN);
2138 vsi->link_active = FALSE;
2146 ixl_stop(struct ixl_pf *pf)
2149 ixl_stop_locked(pf);
2152 ixl_teardown_queue_msix(&pf->vsi);
2153 ixl_free_queue_tqs(&pf->vsi);
2156 /*********************************************************************
2158 * This routine disables all traffic on the adapter by issuing a
2159 * global reset on the MAC and deallocates TX/RX buffers.
2161 **********************************************************************/
2164 ixl_stop_locked(struct ixl_pf *pf)
2166 struct ixl_vsi *vsi = &pf->vsi;
2167 struct ifnet *ifp = vsi->ifp;
2169 INIT_DEBUGOUT("ixl_stop: begin\n");
2171 IXL_PF_LOCK_ASSERT(pf);
2173 /* Stop the local timer */
2174 callout_stop(&pf->timer);
2176 ixl_disable_rings_intr(vsi);
2177 ixl_disable_rings(vsi);
2179 /* Tell the stack that the interface is no longer active */
2180 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING);
2184 /*********************************************************************
2186 * Setup MSIX Interrupt resources and handlers for the VSI
2188 **********************************************************************/
2190 ixl_assign_vsi_legacy(struct ixl_pf *pf)
2192 device_t dev = pf->dev;
2193 struct ixl_vsi *vsi = &pf->vsi;
2194 struct ixl_queue *que = vsi->queues;
2199 pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2200 &rid, RF_SHAREABLE | RF_ACTIVE);
2201 if (pf->res == NULL) {
2202 device_printf(dev, "Unable to allocate"
2203 " bus resource: vsi legacy/msi interrupt\n");
2207 /* Set the handler function */
2208 error = bus_setup_intr(dev, pf->res,
2209 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2210 ixl_intr, pf, &pf->tag);
2213 device_printf(dev, "Failed to register legacy/msi handler\n");
2216 bus_describe_intr(dev, pf->res, pf->tag, "irq0");
2217 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2218 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2219 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2220 taskqueue_thread_enqueue, &que->tq);
2221 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2222 device_get_nameunit(dev));
2223 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2225 pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2226 taskqueue_thread_enqueue, &pf->tq);
2227 taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2228 device_get_nameunit(dev));
2234 ixl_setup_adminq_tq(struct ixl_pf *pf)
2236 device_t dev = pf->dev;
2239 /* Tasklet for Admin Queue interrupts */
2240 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2243 TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2245 /* Create and start Admin Queue taskqueue */
2246 pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT,
2247 taskqueue_thread_enqueue, &pf->tq);
2249 device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n");
2252 error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq",
2253 device_get_nameunit(dev));
2255 device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n",
2257 taskqueue_free(pf->tq);
2264 ixl_setup_queue_tqs(struct ixl_vsi *vsi)
2266 struct ixl_queue *que = vsi->queues;
2267 device_t dev = vsi->dev;
2273 /* Create queue tasks and start queue taskqueues */
2274 for (int i = 0; i < vsi->num_queues; i++, que++) {
2275 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2276 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2277 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2278 taskqueue_thread_enqueue, &que->tq);
2280 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2281 CPU_SETOF(cpu_id, &cpu_mask);
2282 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2283 &cpu_mask, "%s (bucket %d)",
2284 device_get_nameunit(dev), cpu_id);
2286 taskqueue_start_threads(&que->tq, 1, PI_NET,
2287 "%s (que %d)", device_get_nameunit(dev), que->me);
2295 ixl_free_adminq_tq(struct ixl_pf *pf)
2298 taskqueue_free(pf->tq);
2304 ixl_free_queue_tqs(struct ixl_vsi *vsi)
2306 struct ixl_queue *que = vsi->queues;
2308 for (int i = 0; i < vsi->num_queues; i++, que++) {
2310 taskqueue_free(que->tq);
2317 ixl_setup_adminq_msix(struct ixl_pf *pf)
2319 device_t dev = pf->dev;
2322 /* Admin IRQ rid is 1, vector is 0 */
2324 /* Get interrupt resource from bus */
2325 pf->res = bus_alloc_resource_any(dev,
2326 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2328 device_printf(dev, "bus_alloc_resource_any() for Admin Queue"
2329 " interrupt failed [rid=%d]\n", rid);
2332 /* Then associate interrupt with handler */
2333 error = bus_setup_intr(dev, pf->res,
2334 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2335 ixl_msix_adminq, pf, &pf->tag);
2338 device_printf(dev, "bus_setup_intr() for Admin Queue"
2339 " interrupt handler failed, error %d\n", error);
2342 error = bus_describe_intr(dev, pf->res, pf->tag, "aq");
2344 /* Probably non-fatal? */
2345 device_printf(dev, "bus_describe_intr() for Admin Queue"
2346 " interrupt name failed, error %d\n", error);
2354 * Allocate interrupt resources from bus and associate an interrupt handler
2355 * to those for the VSI's queues.
2358 ixl_setup_queue_msix(struct ixl_vsi *vsi)
2360 device_t dev = vsi->dev;
2361 struct ixl_queue *que = vsi->queues;
2362 struct tx_ring *txr;
2363 int error, rid, vector = 1;
2365 /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */
2366 for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2370 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2371 RF_SHAREABLE | RF_ACTIVE);
2373 device_printf(dev, "bus_alloc_resource_any() for"
2374 " Queue %d interrupt failed [rid=%d]\n",
2378 /* Set the handler function */
2379 error = bus_setup_intr(dev, que->res,
2380 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2381 ixl_msix_que, que, &que->tag);
2383 device_printf(dev, "bus_setup_intr() for Queue %d"
2384 " interrupt handler failed, error %d\n",
2386 // TODO: Check for error from this?
2387 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2390 error = bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2392 device_printf(dev, "bus_describe_intr() for Queue %d"
2393 " interrupt name failed, error %d\n",
2396 /* Bind the vector to a CPU */
2398 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2400 error = bus_bind_intr(dev, que->res, cpu_id);
2402 device_printf(dev, "bus_bind_intr() for Queue %d"
2403 " to CPU %d failed, error %d\n",
2404 que->me, cpu_id, error);
2414 * Allocate MSI/X vectors
2417 ixl_init_msix(struct ixl_pf *pf)
2419 device_t dev = pf->dev;
2420 int rid, want, vectors, queues, available;
2422 /* Override by tuneable */
2423 if (ixl_enable_msix == 0)
2427 ** When used in a virtualized environment
2428 ** PCI BUSMASTER capability may not be set
2429 ** so explicity set it here and rewrite
2430 ** the ENABLE in the MSIX control register
2431 ** at this point to cause the host to
2432 ** successfully initialize us.
2437 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2438 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2439 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2440 pci_find_cap(dev, PCIY_MSIX, &rid);
2441 rid += PCIR_MSIX_CTRL;
2442 msix_ctrl = pci_read_config(dev, rid, 2);
2443 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2444 pci_write_config(dev, rid, msix_ctrl, 2);
2447 /* First try MSI/X */
2448 rid = PCIR_BAR(IXL_BAR);
2449 pf->msix_mem = bus_alloc_resource_any(dev,
2450 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2451 if (!pf->msix_mem) {
2452 /* May not be enabled */
2453 device_printf(pf->dev,
2454 "Unable to map MSIX table\n");
2458 available = pci_msix_count(dev);
2459 if (available == 0) { /* system has msix disabled */
2460 bus_release_resource(dev, SYS_RES_MEMORY,
2462 pf->msix_mem = NULL;
2466 /* Figure out a reasonable auto config value */
2467 queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2469 /* Override with tunable value if tunable is less than autoconfig count */
2470 if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2471 queues = ixl_max_queues;
2472 else if ((ixl_max_queues != 0) && (ixl_max_queues > queues))
2473 device_printf(dev, "ixl_max_queues > # of cpus, using "
2474 "autoconfig amount...\n");
2475 /* Or limit maximum auto-configured queues to 8 */
2476 else if ((ixl_max_queues == 0) && (queues > 8))
2480 /* If we're doing RSS, clamp at the number of RSS buckets */
2481 if (queues > rss_getnumbuckets())
2482 queues = rss_getnumbuckets();
2486 ** Want one vector (RX/TX pair) per queue
2487 ** plus an additional for the admin queue.
2490 if (want <= available) /* Have enough */
2493 device_printf(pf->dev,
2494 "MSIX Configuration Problem, "
2495 "%d vectors available but %d wanted!\n",
2497 return (0); /* Will go to Legacy setup */
2500 if (pci_alloc_msix(dev, &vectors) == 0) {
2501 device_printf(pf->dev,
2502 "Using MSIX interrupts with %d vectors\n", vectors);
2504 pf->vsi.num_queues = queues;
2507 * If we're doing RSS, the number of queues needs to
2508 * match the number of RSS buckets that are configured.
2510 * + If there's more queues than RSS buckets, we'll end
2511 * up with queues that get no traffic.
2513 * + If there's more RSS buckets than queues, we'll end
2514 * up having multiple RSS buckets map to the same queue,
2515 * so there'll be some contention.
2517 if (queues != rss_getnumbuckets()) {
2519 "%s: queues (%d) != RSS buckets (%d)"
2520 "; performance will be impacted.\n",
2521 __func__, queues, rss_getnumbuckets());
2527 vectors = pci_msi_count(dev);
2528 pf->vsi.num_queues = 1;
2530 ixl_enable_msix = 0;
2531 if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2532 device_printf(pf->dev, "Using an MSI interrupt\n");
2535 device_printf(pf->dev, "Using a Legacy interrupt\n");
2541 * Configure admin queue/misc interrupt cause registers in hardware.
2544 ixl_configure_intr0_msix(struct ixl_pf *pf)
2546 struct i40e_hw *hw = &pf->hw;
2549 /* First set up the adminq - vector 0 */
2550 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
2551 rd32(hw, I40E_PFINT_ICR0); /* read to clear */
2553 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2554 I40E_PFINT_ICR0_ENA_GRST_MASK |
2555 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
2556 I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2557 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2558 I40E_PFINT_ICR0_ENA_VFLR_MASK |
2559 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2560 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2563 * 0x7FF is the end of the queue list.
2564 * This means we won't use MSI-X vector 0 for a queue interrupt
2567 wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2568 /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */
2569 wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E);
2571 wr32(hw, I40E_PFINT_DYN_CTL0,
2572 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2573 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2575 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2579 * Configure queue interrupt cause registers in hardware.
2582 ixl_configure_queue_intr_msix(struct ixl_pf *pf)
2584 struct i40e_hw *hw = &pf->hw;
2585 struct ixl_vsi *vsi = &pf->vsi;
2589 for (int i = 0; i < vsi->num_queues; i++, vector++) {
2590 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
2591 /* First queue type is RX / 0 */
2592 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2594 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2595 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2596 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2597 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2598 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2599 wr32(hw, I40E_QINT_RQCTL(i), reg);
2601 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2602 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2603 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2604 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2605 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2606 wr32(hw, I40E_QINT_TQCTL(i), reg);
2611 * Configure for MSI single vector operation
2614 ixl_configure_legacy(struct ixl_pf *pf)
2616 struct i40e_hw *hw = &pf->hw;
2619 wr32(hw, I40E_PFINT_ITR0(0), 0);
2620 wr32(hw, I40E_PFINT_ITR0(1), 0);
2622 /* Setup "other" causes */
2623 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2624 | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2625 | I40E_PFINT_ICR0_ENA_GRST_MASK
2626 | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2627 | I40E_PFINT_ICR0_ENA_GPIO_MASK
2628 | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2629 | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2630 | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2631 | I40E_PFINT_ICR0_ENA_VFLR_MASK
2632 | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2634 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2636 /* SW_ITR_IDX = 0, but don't change INTENA */
2637 wr32(hw, I40E_PFINT_DYN_CTL0,
2638 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2639 I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2640 /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2641 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2643 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2644 wr32(hw, I40E_PFINT_LNKLST0, 0);
2646 /* Associate the queue pair to the vector and enable the q int */
2647 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2648 | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2649 | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2650 wr32(hw, I40E_QINT_RQCTL(0), reg);
2652 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2653 | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2654 | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2655 wr32(hw, I40E_QINT_TQCTL(0), reg);
2660 * Get initial ITR values from tunable values.
2663 ixl_configure_itr(struct ixl_pf *pf)
2665 struct i40e_hw *hw = &pf->hw;
2666 struct ixl_vsi *vsi = &pf->vsi;
2667 struct ixl_queue *que = vsi->queues;
2669 vsi->rx_itr_setting = ixl_rx_itr;
2670 vsi->tx_itr_setting = ixl_tx_itr;
2672 for (int i = 0; i < vsi->num_queues; i++, que++) {
2673 struct tx_ring *txr = &que->txr;
2674 struct rx_ring *rxr = &que->rxr;
2676 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2677 vsi->rx_itr_setting);
2678 rxr->itr = vsi->rx_itr_setting;
2679 rxr->latency = IXL_AVE_LATENCY;
2681 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2682 vsi->tx_itr_setting);
2683 txr->itr = vsi->tx_itr_setting;
2684 txr->latency = IXL_AVE_LATENCY;
2690 ixl_allocate_pci_resources(struct ixl_pf *pf)
2693 device_t dev = pf->dev;
2696 pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2699 if (!(pf->pci_mem)) {
2700 device_printf(dev, "Unable to allocate bus resource: PCI memory\n");
2704 pf->osdep.mem_bus_space_tag =
2705 rman_get_bustag(pf->pci_mem);
2706 pf->osdep.mem_bus_space_handle =
2707 rman_get_bushandle(pf->pci_mem);
2708 pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2709 pf->osdep.flush_reg = I40E_GLGEN_STAT;
2710 pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2712 pf->hw.back = &pf->osdep;
2715 ** Now setup MSI or MSI/X, should
2716 ** return us the number of supported
2717 ** vectors. (Will be 1 for MSI)
2719 pf->msix = ixl_init_msix(pf);
2724 * Teardown and release the admin queue/misc vector
2728 ixl_teardown_adminq_msix(struct ixl_pf *pf)
2730 device_t dev = pf->dev;
2733 if (pf->admvec) /* we are doing MSIX */
2734 rid = pf->admvec + 1;
2736 (pf->msix != 0) ? (rid = 1):(rid = 0);
2738 // TODO: Check for errors from bus_teardown_intr
2739 // TODO: Check for errors from bus_release_resource
2740 if (pf->tag != NULL) {
2741 bus_teardown_intr(dev, pf->res, pf->tag);
2744 if (pf->res != NULL) {
2745 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2753 ixl_teardown_queue_msix(struct ixl_vsi *vsi)
2755 struct ixl_queue *que = vsi->queues;
2756 device_t dev = vsi->dev;
2759 /* We may get here before stations are setup */
2760 if ((!ixl_enable_msix) || (que == NULL))
2763 /* Release all MSIX queue resources */
2764 for (int i = 0; i < vsi->num_queues; i++, que++) {
2765 rid = que->msix + 1;
2766 if (que->tag != NULL) {
2767 error = bus_teardown_intr(dev, que->res, que->tag);
2769 device_printf(dev, "bus_teardown_intr() for"
2770 " Queue %d interrupt failed\n",
2776 if (que->res != NULL) {
2777 error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2779 device_printf(dev, "bus_release_resource() for"
2780 " Queue %d interrupt failed [rid=%d]\n",
2792 ixl_free_pci_resources(struct ixl_pf *pf)
2794 device_t dev = pf->dev;
2797 ixl_teardown_queue_msix(&pf->vsi);
2798 ixl_teardown_adminq_msix(pf);
2801 pci_release_msi(dev);
2803 memrid = PCIR_BAR(IXL_BAR);
2805 if (pf->msix_mem != NULL)
2806 bus_release_resource(dev, SYS_RES_MEMORY,
2807 memrid, pf->msix_mem);
2809 if (pf->pci_mem != NULL)
2810 bus_release_resource(dev, SYS_RES_MEMORY,
2811 PCIR_BAR(0), pf->pci_mem);
2817 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2819 /* Display supported media types */
2820 if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2821 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2823 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2824 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2825 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2826 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2827 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2828 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2830 if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2831 phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2832 phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2833 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2835 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2836 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2837 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2838 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2839 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2840 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2842 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2843 phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2844 phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2845 phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2846 phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2847 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2848 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2849 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2850 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2851 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2853 #ifndef IFM_ETH_XTYPE
2854 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2855 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2857 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2858 phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2859 phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2860 phy_type & (1 << I40E_PHY_TYPE_SFI))
2861 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2862 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2863 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2864 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2865 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2867 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2868 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2869 if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2870 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2872 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2873 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2875 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2876 || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2877 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2878 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2879 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2880 if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2881 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2882 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2883 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2884 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2885 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2887 if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2888 ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2890 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2891 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2892 if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2893 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2897 /*********************************************************************
2899 * Setup networking device structure and register an interface.
2901 **********************************************************************/
2903 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2906 struct i40e_hw *hw = vsi->hw;
2907 struct ixl_queue *que = vsi->queues;
2908 struct i40e_aq_get_phy_abilities_resp abilities;
2909 enum i40e_status_code aq_error = 0;
2911 INIT_DEBUGOUT("ixl_setup_interface: begin");
2913 ifp = vsi->ifp = if_alloc(IFT_ETHER);
2915 device_printf(dev, "can not allocate ifnet structure\n");
2918 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2919 ifp->if_mtu = ETHERMTU;
2920 ifp->if_baudrate = IF_Gbps(40);
2921 ifp->if_init = ixl_init;
2922 ifp->if_softc = vsi;
2923 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2924 ifp->if_ioctl = ixl_ioctl;
2926 #if __FreeBSD_version >= 1100036
2927 if_setgetcounterfn(ifp, ixl_get_counter);
2930 ifp->if_transmit = ixl_mq_start;
2932 ifp->if_qflush = ixl_qflush;
2934 ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2936 vsi->max_frame_size =
2937 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2938 + ETHER_VLAN_ENCAP_LEN;
2941 * Tell the upper layer(s) we support long frames.
2943 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2945 ifp->if_capabilities |= IFCAP_HWCSUM;
2946 ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2947 ifp->if_capabilities |= IFCAP_TSO;
2948 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2949 ifp->if_capabilities |= IFCAP_LRO;
2951 /* VLAN capabilties */
2952 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2955 | IFCAP_VLAN_HWCSUM;
2956 ifp->if_capenable = ifp->if_capabilities;
2959 ** Don't turn this on by default, if vlans are
2960 ** created on another pseudo device (eg. lagg)
2961 ** then vlan events are not passed thru, breaking
2962 ** operation, but with HW FILTER off it works. If
2963 ** using vlans directly on the ixl driver you can
2964 ** enable this and get full hardware tag filtering.
2966 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2969 * Specify the media types supported by this adapter and register
2970 * callbacks to update media and link information
2972 ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2975 aq_error = i40e_aq_get_phy_capabilities(hw,
2976 FALSE, TRUE, &abilities, NULL);
2977 /* May need delay to detect fiber correctly */
2978 if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2979 i40e_msec_delay(200);
2980 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2981 TRUE, &abilities, NULL);
2984 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2985 device_printf(dev, "Unknown PHY type detected!\n");
2988 "Error getting supported media types, err %d,"
2989 " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2993 ixl_add_ifmedia(vsi, abilities.phy_type);
2995 /* Use autoselect media by default */
2996 ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2997 ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2999 ether_ifattach(ifp, hw->mac.addr);
3005 ** Run when the Admin Queue gets a link state change interrupt.
3008 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
3010 struct i40e_hw *hw = &pf->hw;
3011 device_t dev = pf->dev;
3012 struct i40e_aqc_get_link_status *status =
3013 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
3015 /* Request link status from adapter */
3016 hw->phy.get_link_info = TRUE;
3017 i40e_get_link_status(hw, &pf->link_up);
3019 /* Print out message if an unqualified module is found */
3020 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
3021 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
3022 (!(status->link_info & I40E_AQ_LINK_UP)))
3023 device_printf(dev, "Link failed because "
3024 "an unqualified module was detected!\n");
3026 /* Update OS link info */
3027 ixl_update_link_status(pf);
3030 /*********************************************************************
3032 * Get Firmware Switch configuration
3033 * - this will need to be more robust when more complex
3034 * switch configurations are enabled.
3036 **********************************************************************/
3038 ixl_switch_config(struct ixl_pf *pf)
3040 struct i40e_hw *hw = &pf->hw;
3041 struct ixl_vsi *vsi = &pf->vsi;
3042 device_t dev = vsi->dev;
3043 struct i40e_aqc_get_switch_config_resp *sw_config;
3044 u8 aq_buf[I40E_AQ_LARGE_BUF];
3048 memset(&aq_buf, 0, sizeof(aq_buf));
3049 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
3050 ret = i40e_aq_get_switch_config(hw, sw_config,
3051 sizeof(aq_buf), &next, NULL);
3053 device_printf(dev, "aq_get_switch_config() failed, error %d,"
3054 " aq_error %d\n", ret, pf->hw.aq.asq_last_status);
3059 "Switch config: header reported: %d in structure, %d total\n",
3060 sw_config->header.num_reported, sw_config->header.num_total);
3061 for (int i = 0; i < sw_config->header.num_reported; i++) {
3063 "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
3064 sw_config->element[i].element_type,
3065 sw_config->element[i].seid,
3066 sw_config->element[i].uplink_seid,
3067 sw_config->element[i].downlink_seid);
3070 /* Simplified due to a single VSI at the moment */
3071 vsi->uplink_seid = sw_config->element[0].uplink_seid;
3072 vsi->downlink_seid = sw_config->element[0].downlink_seid;
3073 vsi->seid = sw_config->element[0].seid;
3077 /*********************************************************************
3079 * Initialize the VSI: this handles contexts, which means things
3080 * like the number of descriptors, buffer size,
3081 * plus we init the rings thru this function.
3083 **********************************************************************/
3085 ixl_initialize_vsi(struct ixl_vsi *vsi)
3087 struct ixl_pf *pf = vsi->back;
3088 struct ixl_queue *que = vsi->queues;
3089 device_t dev = vsi->dev;
3090 struct i40e_hw *hw = vsi->hw;
3091 struct i40e_vsi_context ctxt;
3094 memset(&ctxt, 0, sizeof(ctxt));
3095 ctxt.seid = vsi->seid;
3096 if (pf->veb_seid != 0)
3097 ctxt.uplink_seid = pf->veb_seid;
3098 ctxt.pf_num = hw->pf_id;
3099 err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
3101 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d"
3102 " aq_error %d\n", err, hw->aq.asq_last_status);
3106 device_printf(dev, "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
3107 "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
3108 "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
3109 ctxt.uplink_seid, ctxt.vsi_number,
3110 ctxt.vsis_allocated, ctxt.vsis_unallocated,
3111 ctxt.flags, ctxt.pf_num, ctxt.vf_num,
3112 ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
3115 ** Set the queue and traffic class bits
3116 ** - when multiple traffic classes are supported
3117 ** this will need to be more robust.
3119 ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
3120 ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
3121 /* In contig mode, que_mapping[0] is first queue index used by this VSI */
3122 ctxt.info.queue_mapping[0] = 0;
3124 * This VSI will only use traffic class 0; start traffic class 0's
3125 * queue allocation at queue 0, and assign it 64 (2^6) queues (though
3126 * the driver may not use all of them).
3128 ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
3129 & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
3130 ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
3131 & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
3133 /* Set VLAN receive stripping mode */
3134 ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
3135 ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
3136 if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
3137 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3139 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3141 /* Keep copy of VSI info in VSI for statistic counters */
3142 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
3144 /* Reset VSI statistics */
3145 ixl_vsi_reset_stats(vsi);
3146 vsi->hw_filters_add = 0;
3147 vsi->hw_filters_del = 0;
3149 ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
3151 err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3153 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d, aq_error %d\n",
3154 err, hw->aq.asq_last_status);
3158 for (int i = 0; i < vsi->num_queues; i++, que++) {
3159 struct tx_ring *txr = &que->txr;
3160 struct rx_ring *rxr = &que->rxr;
3161 struct i40e_hmc_obj_txq tctx;
3162 struct i40e_hmc_obj_rxq rctx;
3166 /* Setup the HMC TX Context */
3167 size = que->num_desc * sizeof(struct i40e_tx_desc);
3168 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
3169 tctx.new_context = 1;
3170 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
3171 tctx.qlen = que->num_desc;
3173 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
3174 /* Enable HEAD writeback */
3175 tctx.head_wb_ena = 1;
3176 tctx.head_wb_addr = txr->dma.pa +
3177 (que->num_desc * sizeof(struct i40e_tx_desc));
3178 tctx.rdylist_act = 0;
3179 err = i40e_clear_lan_tx_queue_context(hw, i);
3181 device_printf(dev, "Unable to clear TX context\n");
3184 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
3186 device_printf(dev, "Unable to set TX context\n");
3189 /* Associate the ring with this PF */
3190 txctl = I40E_QTX_CTL_PF_QUEUE;
3191 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3192 I40E_QTX_CTL_PF_INDX_MASK);
3193 wr32(hw, I40E_QTX_CTL(i), txctl);
3196 /* Do ring (re)init */
3197 ixl_init_tx_ring(que);
3199 /* Next setup the HMC RX Context */
3200 if (vsi->max_frame_size <= MCLBYTES)
3201 rxr->mbuf_sz = MCLBYTES;
3203 rxr->mbuf_sz = MJUMPAGESIZE;
3205 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
3207 /* Set up an RX context for the HMC */
3208 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
3209 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
3210 /* ignore header split for now */
3211 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
3212 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
3213 vsi->max_frame_size : max_rxmax;
3215 rctx.dsize = 1; /* do 32byte descriptors */
3216 rctx.hsplit_0 = 0; /* no HDR split initially */
3217 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
3218 rctx.qlen = que->num_desc;
3219 rctx.tphrdesc_ena = 1;
3220 rctx.tphwdesc_ena = 1;
3221 rctx.tphdata_ena = 0;
3222 rctx.tphhead_ena = 0;
3223 rctx.lrxqthresh = 2;
3230 err = i40e_clear_lan_rx_queue_context(hw, i);
3233 "Unable to clear RX context %d\n", i);
3236 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
3238 device_printf(dev, "Unable to set RX context %d\n", i);
3241 err = ixl_init_rx_ring(que);
3243 device_printf(dev, "Fail in init_rx_ring %d\n", i);
3247 /* preserve queue */
3248 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
3249 struct netmap_adapter *na = NA(vsi->ifp);
3250 struct netmap_kring *kring = &na->rx_rings[i];
3251 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
3252 wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
3254 #endif /* DEV_NETMAP */
3255 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
3261 /*********************************************************************
3263 * Free all VSI structs.
3265 **********************************************************************/
3267 ixl_free_vsi(struct ixl_vsi *vsi)
3269 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3270 struct ixl_queue *que = vsi->queues;
3272 /* Free station queues */
3276 for (int i = 0; i < vsi->num_queues; i++, que++) {
3277 struct tx_ring *txr = &que->txr;
3278 struct rx_ring *rxr = &que->rxr;
3280 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
3283 ixl_free_que_tx(que);
3285 i40e_free_dma_mem(&pf->hw, &txr->dma);
3287 IXL_TX_LOCK_DESTROY(txr);
3289 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
3292 ixl_free_que_rx(que);
3294 i40e_free_dma_mem(&pf->hw, &rxr->dma);
3296 IXL_RX_LOCK_DESTROY(rxr);
3299 free(vsi->queues, M_DEVBUF);
3302 /* Free VSI filter list */
3303 ixl_free_mac_filters(vsi);
3307 ixl_free_mac_filters(struct ixl_vsi *vsi)
3309 struct ixl_mac_filter *f;
3311 while (!SLIST_EMPTY(&vsi->ftl)) {
3312 f = SLIST_FIRST(&vsi->ftl);
3313 SLIST_REMOVE_HEAD(&vsi->ftl, next);
3319 /*********************************************************************
3321 * Allocate memory for the VSI (virtual station interface) and their
3322 * associated queues, rings and the descriptors associated with each,
3323 * called only once at attach.
3325 **********************************************************************/
3327 ixl_setup_stations(struct ixl_pf *pf)
3329 device_t dev = pf->dev;
3330 struct ixl_vsi *vsi;
3331 struct ixl_queue *que;
3332 struct tx_ring *txr;
3333 struct rx_ring *rxr;
3335 int error = I40E_SUCCESS;
3338 vsi->back = (void *)pf;
3344 /* Get memory for the station queues */
3346 (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
3347 vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3348 device_printf(dev, "Unable to allocate queue memory\n");
3353 for (int i = 0; i < vsi->num_queues; i++) {
3354 que = &vsi->queues[i];
3355 que->num_desc = ixl_ringsz;
3358 /* mark the queue as active */
3359 vsi->active_queues |= (u64)1 << que->me;
3362 txr->tail = I40E_QTX_TAIL(que->me);
3364 /* Initialize the TX lock */
3365 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3366 device_get_nameunit(dev), que->me);
3367 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
3368 /* Create the TX descriptor ring */
3369 tsize = roundup2((que->num_desc *
3370 sizeof(struct i40e_tx_desc)) +
3371 sizeof(u32), DBA_ALIGN);
3372 if (i40e_allocate_dma_mem(&pf->hw,
3373 &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
3375 "Unable to allocate TX Descriptor memory\n");
3379 txr->base = (struct i40e_tx_desc *)txr->dma.va;
3380 bzero((void *)txr->base, tsize);
3381 /* Now allocate transmit soft structs for the ring */
3382 if (ixl_allocate_tx_data(que)) {
3384 "Critical Failure setting up TX structures\n");
3388 /* Allocate a buf ring */
3389 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3390 M_NOWAIT, &txr->mtx);
3391 if (txr->br == NULL) {
3393 "Critical Failure setting up TX buf ring\n");
3399 * Next the RX queues...
3401 rsize = roundup2(que->num_desc *
3402 sizeof(union i40e_rx_desc), DBA_ALIGN);
3405 rxr->tail = I40E_QRX_TAIL(que->me);
3407 /* Initialize the RX side lock */
3408 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3409 device_get_nameunit(dev), que->me);
3410 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3412 if (i40e_allocate_dma_mem(&pf->hw,
3413 &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3415 "Unable to allocate RX Descriptor memory\n");
3419 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3420 bzero((void *)rxr->base, rsize);
3422 /* Allocate receive soft structs for the ring*/
3423 if (ixl_allocate_rx_data(que)) {
3425 "Critical Failure setting up receive structs\n");
3434 for (int i = 0; i < vsi->num_queues; i++) {
3435 que = &vsi->queues[i];
3439 i40e_free_dma_mem(&pf->hw, &rxr->dma);
3441 i40e_free_dma_mem(&pf->hw, &txr->dma);
3449 ** Provide a update to the queue RX
3450 ** interrupt moderation value.
3453 ixl_set_queue_rx_itr(struct ixl_queue *que)
3455 struct ixl_vsi *vsi = que->vsi;
3456 struct i40e_hw *hw = vsi->hw;
3457 struct rx_ring *rxr = &que->rxr;
3462 /* Idle, do nothing */
3463 if (rxr->bytes == 0)
3466 if (ixl_dynamic_rx_itr) {
3467 rx_bytes = rxr->bytes/rxr->itr;
3470 /* Adjust latency range */
3471 switch (rxr->latency) {
3472 case IXL_LOW_LATENCY:
3473 if (rx_bytes > 10) {
3474 rx_latency = IXL_AVE_LATENCY;
3475 rx_itr = IXL_ITR_20K;
3478 case IXL_AVE_LATENCY:
3479 if (rx_bytes > 20) {
3480 rx_latency = IXL_BULK_LATENCY;
3481 rx_itr = IXL_ITR_8K;
3482 } else if (rx_bytes <= 10) {
3483 rx_latency = IXL_LOW_LATENCY;
3484 rx_itr = IXL_ITR_100K;
3487 case IXL_BULK_LATENCY:
3488 if (rx_bytes <= 20) {
3489 rx_latency = IXL_AVE_LATENCY;
3490 rx_itr = IXL_ITR_20K;
3495 rxr->latency = rx_latency;
3497 if (rx_itr != rxr->itr) {
3498 /* do an exponential smoothing */
3499 rx_itr = (10 * rx_itr * rxr->itr) /
3500 ((9 * rx_itr) + rxr->itr);
3501 rxr->itr = rx_itr & IXL_MAX_ITR;
3502 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3503 que->me), rxr->itr);
3505 } else { /* We may have have toggled to non-dynamic */
3506 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3507 vsi->rx_itr_setting = ixl_rx_itr;
3508 /* Update the hardware if needed */
3509 if (rxr->itr != vsi->rx_itr_setting) {
3510 rxr->itr = vsi->rx_itr_setting;
3511 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3512 que->me), rxr->itr);
3522 ** Provide a update to the queue TX
3523 ** interrupt moderation value.
3526 ixl_set_queue_tx_itr(struct ixl_queue *que)
3528 struct ixl_vsi *vsi = que->vsi;
3529 struct i40e_hw *hw = vsi->hw;
3530 struct tx_ring *txr = &que->txr;
3536 /* Idle, do nothing */
3537 if (txr->bytes == 0)
3540 if (ixl_dynamic_tx_itr) {
3541 tx_bytes = txr->bytes/txr->itr;
3544 switch (txr->latency) {
3545 case IXL_LOW_LATENCY:
3546 if (tx_bytes > 10) {
3547 tx_latency = IXL_AVE_LATENCY;
3548 tx_itr = IXL_ITR_20K;
3551 case IXL_AVE_LATENCY:
3552 if (tx_bytes > 20) {
3553 tx_latency = IXL_BULK_LATENCY;
3554 tx_itr = IXL_ITR_8K;
3555 } else if (tx_bytes <= 10) {
3556 tx_latency = IXL_LOW_LATENCY;
3557 tx_itr = IXL_ITR_100K;
3560 case IXL_BULK_LATENCY:
3561 if (tx_bytes <= 20) {
3562 tx_latency = IXL_AVE_LATENCY;
3563 tx_itr = IXL_ITR_20K;
3568 txr->latency = tx_latency;
3570 if (tx_itr != txr->itr) {
3571 /* do an exponential smoothing */
3572 tx_itr = (10 * tx_itr * txr->itr) /
3573 ((9 * tx_itr) + txr->itr);
3574 txr->itr = tx_itr & IXL_MAX_ITR;
3575 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3576 que->me), txr->itr);
3579 } else { /* We may have have toggled to non-dynamic */
3580 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3581 vsi->tx_itr_setting = ixl_tx_itr;
3582 /* Update the hardware if needed */
3583 if (txr->itr != vsi->tx_itr_setting) {
3584 txr->itr = vsi->tx_itr_setting;
3585 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3586 que->me), txr->itr);
3594 #define QUEUE_NAME_LEN 32
3597 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3598 struct sysctl_ctx_list *ctx, const char *sysctl_name)
3600 struct sysctl_oid *tree;
3601 struct sysctl_oid_list *child;
3602 struct sysctl_oid_list *vsi_list;
3604 tree = device_get_sysctl_tree(pf->dev);
3605 child = SYSCTL_CHILDREN(tree);
3606 vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3607 CTLFLAG_RD, NULL, "VSI Number");
3608 vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3610 ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3615 * ixl_sysctl_qtx_tail_handler
3616 * Retrieves I40E_QTX_TAIL value from hardware
3620 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)
3622 struct ixl_queue *que;
3626 que = ((struct ixl_queue *)oidp->oid_arg1);
3629 val = rd32(que->vsi->hw, que->txr.tail);
3630 error = sysctl_handle_int(oidp, &val, 0, req);
3631 if (error || !req->newptr)
3637 * ixl_sysctl_qrx_tail_handler
3638 * Retrieves I40E_QRX_TAIL value from hardware
3642 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)
3644 struct ixl_queue *que;
3648 que = ((struct ixl_queue *)oidp->oid_arg1);
3651 val = rd32(que->vsi->hw, que->rxr.tail);
3652 error = sysctl_handle_int(oidp, &val, 0, req);
3653 if (error || !req->newptr)
3660 ixl_add_hw_stats(struct ixl_pf *pf)
3662 device_t dev = pf->dev;
3663 struct ixl_vsi *vsi = &pf->vsi;
3664 struct ixl_queue *queues = vsi->queues;
3665 struct i40e_hw_port_stats *pf_stats = &pf->stats;
3667 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3668 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3669 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3670 struct sysctl_oid_list *vsi_list;
3672 struct sysctl_oid *queue_node;
3673 struct sysctl_oid_list *queue_list;
3675 struct tx_ring *txr;
3676 struct rx_ring *rxr;
3677 char queue_namebuf[QUEUE_NAME_LEN];
3679 /* Driver statistics */
3680 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3681 CTLFLAG_RD, &pf->watchdog_events,
3682 "Watchdog timeouts");
3683 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3684 CTLFLAG_RD, &pf->admin_irq,
3685 "Admin Queue IRQ Handled");
3687 ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3688 vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3690 /* Queue statistics */
3691 for (int q = 0; q < vsi->num_queues; q++) {
3692 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3693 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3694 OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3695 queue_list = SYSCTL_CHILDREN(queue_node);
3697 txr = &(queues[q].txr);
3698 rxr = &(queues[q].rxr);
3700 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3701 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3702 "m_defrag() failed");
3703 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3704 CTLFLAG_RD, &(queues[q].irqs),
3705 "irqs on this queue");
3706 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3707 CTLFLAG_RD, &(queues[q].tso),
3709 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3710 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3711 "Driver tx dma failure in xmit");
3712 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3713 CTLFLAG_RD, &(txr->no_desc),
3714 "Queue No Descriptor Available");
3715 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3716 CTLFLAG_RD, &(txr->total_packets),
3717 "Queue Packets Transmitted");
3718 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3719 CTLFLAG_RD, &(txr->tx_bytes),
3720 "Queue Bytes Transmitted");
3721 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3722 CTLFLAG_RD, &(rxr->rx_packets),
3723 "Queue Packets Received");
3724 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3725 CTLFLAG_RD, &(rxr->rx_bytes),
3726 "Queue Bytes Received");
3727 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err",
3728 CTLFLAG_RD, &(rxr->desc_errs),
3729 "Queue Rx Descriptor Errors");
3730 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr",
3731 CTLFLAG_RD, &(rxr->itr), 0,
3732 "Queue Rx ITR Interval");
3733 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr",
3734 CTLFLAG_RD, &(txr->itr), 0,
3735 "Queue Tx ITR Interval");
3736 // Not actual latency; just a calculated value to put in a register
3737 // TODO: Put in better descriptions here
3738 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_latency",
3739 CTLFLAG_RD, &(rxr->latency), 0,
3740 "Queue Rx ITRL Average Interval");
3741 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_latency",
3742 CTLFLAG_RD, &(txr->latency), 0,
3743 "Queue Tx ITRL Average Interval");
3746 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done",
3747 CTLFLAG_RD, &(rxr->not_done),
3748 "Queue Rx Descriptors not Done");
3749 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh",
3750 CTLFLAG_RD, &(rxr->next_refresh), 0,
3751 "Queue Rx Descriptors not Done");
3752 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check",
3753 CTLFLAG_RD, &(rxr->next_check), 0,
3754 "Queue Rx Descriptors not Done");
3755 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail",
3756 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3757 sizeof(struct ixl_queue),
3758 ixl_sysctl_qtx_tail_handler, "IU",
3759 "Queue Transmit Descriptor Tail");
3760 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail",
3761 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3762 sizeof(struct ixl_queue),
3763 ixl_sysctl_qrx_tail_handler, "IU",
3764 "Queue Receive Descriptor Tail");
3769 ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3773 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3774 struct sysctl_oid_list *child,
3775 struct i40e_eth_stats *eth_stats)
3777 struct ixl_sysctl_info ctls[] =
3779 {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3780 {ð_stats->rx_unicast, "ucast_pkts_rcvd",
3781 "Unicast Packets Received"},
3782 {ð_stats->rx_multicast, "mcast_pkts_rcvd",
3783 "Multicast Packets Received"},
3784 {ð_stats->rx_broadcast, "bcast_pkts_rcvd",
3785 "Broadcast Packets Received"},
3786 {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3787 {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3788 {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3789 {ð_stats->tx_multicast, "mcast_pkts_txd",
3790 "Multicast Packets Transmitted"},
3791 {ð_stats->tx_broadcast, "bcast_pkts_txd",
3792 "Broadcast Packets Transmitted"},
3797 struct ixl_sysctl_info *entry = ctls;
3798 while (entry->stat != NULL)
3800 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3801 CTLFLAG_RD, entry->stat,
3802 entry->description);
3808 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3809 struct sysctl_oid_list *child,
3810 struct i40e_hw_port_stats *stats)
3812 struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3813 CTLFLAG_RD, NULL, "Mac Statistics");
3814 struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3816 struct i40e_eth_stats *eth_stats = &stats->eth;
3817 ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3819 struct ixl_sysctl_info ctls[] =
3821 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3822 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3823 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3824 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3825 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3826 /* Packet Reception Stats */
3827 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3828 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3829 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3830 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3831 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3832 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3833 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3834 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3835 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3836 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3837 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3838 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3839 /* Packet Transmission Stats */
3840 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3841 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3842 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3843 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3844 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3845 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3846 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3848 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3849 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3850 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3851 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3856 struct ixl_sysctl_info *entry = ctls;
3857 while (entry->stat != NULL)
3859 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3860 CTLFLAG_RD, entry->stat,
3861 entry->description);
3868 ** ixl_config_rss - setup RSS
3869 ** - note this is done for the single vsi
3872 ixl_config_rss(struct ixl_vsi *vsi)
3874 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3875 struct i40e_hw *hw = vsi->hw;
3877 u64 set_hena = 0, hena;
3880 u32 rss_hash_config;
3881 u32 rss_seed[IXL_KEYSZ];
3883 u32 rss_seed[IXL_KEYSZ] = {0x41b01687,
3884 0x183cfd8c, 0xce880440, 0x580cbc3c,
3885 0x35897377, 0x328b25e1, 0x4fa98922,
3886 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3890 /* Fetch the configured RSS key */
3891 rss_getkey((uint8_t *) &rss_seed);
3894 /* Fill out hash function seed */
3895 for (i = 0; i < IXL_KEYSZ; i++)
3896 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3898 /* Enable PCTYPES for RSS: */
3900 rss_hash_config = rss_gethashconfig();
3901 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3902 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3903 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3904 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3905 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3906 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3907 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3908 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3909 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3910 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3911 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3912 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3913 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3914 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3917 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3918 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3919 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3920 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3921 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3922 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3923 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3924 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3925 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3926 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3927 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3929 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
3930 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
3932 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
3933 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3935 /* Populate the LUT with max no. of queues in round robin fashion */
3936 for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3937 if (j == vsi->num_queues)
3941 * Fetch the RSS bucket id for the given indirection entry.
3942 * Cap it at the number of configured buckets (which is
3945 que_id = rss_get_indirection_to_bucket(i);
3946 que_id = que_id % vsi->num_queues;
3950 /* lut = 4-byte sliding window of 4 lut entries */
3951 lut = (lut << 8) | (que_id &
3952 ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3953 /* On i = 3, we have 4 entries in lut; write to the register */
3955 wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3962 ** This routine is run via an vlan config EVENT,
3963 ** it enables us to use the HW Filter table since
3964 ** we can get the vlan id. This just creates the
3965 ** entry in the soft version of the VFTA, init will
3966 ** repopulate the real table.
3969 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3971 struct ixl_vsi *vsi = ifp->if_softc;
3972 struct i40e_hw *hw = vsi->hw;
3973 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3975 if (ifp->if_softc != arg) /* Not our event */
3978 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
3983 ixl_add_filter(vsi, hw->mac.addr, vtag);
3988 ** This routine is run via an vlan
3989 ** unconfig EVENT, remove our entry
3990 ** in the soft vfta.
3993 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3995 struct ixl_vsi *vsi = ifp->if_softc;
3996 struct i40e_hw *hw = vsi->hw;
3997 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
3999 if (ifp->if_softc != arg)
4002 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4007 ixl_del_filter(vsi, hw->mac.addr, vtag);
4012 ** This routine updates vlan filters, called by init
4013 ** it scans the filter table and then updates the hw
4014 ** after a soft reset.
4017 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
4019 struct ixl_mac_filter *f;
4022 if (vsi->num_vlans == 0)
4025 ** Scan the filter list for vlan entries,
4026 ** mark them for addition and then call
4027 ** for the AQ update.
4029 SLIST_FOREACH(f, &vsi->ftl, next) {
4030 if (f->flags & IXL_FILTER_VLAN) {
4038 printf("setup vlan: no filters found!\n");
4041 flags = IXL_FILTER_VLAN;
4042 flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4043 ixl_add_hw_filters(vsi, flags, cnt);
4048 ** Initialize filter list and add filters that the hardware
4049 ** needs to know about.
4051 ** Requires VSI's filter list & seid to be set before calling.
4054 ixl_init_filters(struct ixl_vsi *vsi)
4056 /* Add broadcast address */
4057 ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
4060 * Prevent Tx flow control frames from being sent out by
4061 * non-firmware transmitters.
4063 i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid);
4067 ** This routine adds mulicast filters
4070 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
4072 struct ixl_mac_filter *f;
4074 /* Does one already exist */
4075 f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4079 f = ixl_get_filter(vsi);
4081 printf("WARNING: no filter available!!\n");
4084 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4085 f->vlan = IXL_VLAN_ANY;
4086 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
4093 ixl_reconfigure_filters(struct ixl_vsi *vsi)
4096 ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
4100 ** This routine adds macvlan filters
4103 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4105 struct ixl_mac_filter *f, *tmp;
4109 DEBUGOUT("ixl_add_filter: begin");
4114 /* Does one already exist */
4115 f = ixl_find_filter(vsi, macaddr, vlan);
4119 ** Is this the first vlan being registered, if so we
4120 ** need to remove the ANY filter that indicates we are
4121 ** not in a vlan, and replace that with a 0 filter.
4123 if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
4124 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4126 ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
4127 ixl_add_filter(vsi, macaddr, 0);
4131 f = ixl_get_filter(vsi);
4133 device_printf(dev, "WARNING: no filter available!!\n");
4136 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4138 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4139 if (f->vlan != IXL_VLAN_ANY)
4140 f->flags |= IXL_FILTER_VLAN;
4144 ixl_add_hw_filters(vsi, f->flags, 1);
4149 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4151 struct ixl_mac_filter *f;
4153 f = ixl_find_filter(vsi, macaddr, vlan);
4157 f->flags |= IXL_FILTER_DEL;
4158 ixl_del_hw_filters(vsi, 1);
4161 /* Check if this is the last vlan removal */
4162 if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
4163 /* Switch back to a non-vlan filter */
4164 ixl_del_filter(vsi, macaddr, 0);
4165 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
4171 ** Find the filter with both matching mac addr and vlan id
4173 static struct ixl_mac_filter *
4174 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4176 struct ixl_mac_filter *f;
4179 SLIST_FOREACH(f, &vsi->ftl, next) {
4180 if (!cmp_etheraddr(f->macaddr, macaddr))
4182 if (f->vlan == vlan) {
4194 ** This routine takes additions to the vsi filter
4195 ** table and creates an Admin Queue call to create
4196 ** the filters in the hardware.
4199 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
4201 struct i40e_aqc_add_macvlan_element_data *a, *b;
4202 struct ixl_mac_filter *f;
4211 IXL_PF_LOCK_ASSERT(pf);
4213 a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
4214 M_DEVBUF, M_NOWAIT | M_ZERO);
4216 device_printf(dev, "add_hw_filters failed to get memory\n");
4221 ** Scan the filter list, each time we find one
4222 ** we add it to the admin queue array and turn off
4225 SLIST_FOREACH(f, &vsi->ftl, next) {
4226 if (f->flags == flags) {
4227 b = &a[j]; // a pox on fvl long names :)
4228 bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
4229 if (f->vlan == IXL_VLAN_ANY) {
4231 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
4233 b->vlan_tag = f->vlan;
4236 b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
4237 f->flags &= ~IXL_FILTER_ADD;
4244 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
4246 device_printf(dev, "aq_add_macvlan err %d, "
4247 "aq_error %d\n", err, hw->aq.asq_last_status);
4249 vsi->hw_filters_add += j;
4256 ** This routine takes removals in the vsi filter
4257 ** table and creates an Admin Queue call to delete
4258 ** the filters in the hardware.
4261 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
4263 struct i40e_aqc_remove_macvlan_element_data *d, *e;
4267 struct ixl_mac_filter *f, *f_temp;
4270 DEBUGOUT("ixl_del_hw_filters: begin\n");
4276 d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
4277 M_DEVBUF, M_NOWAIT | M_ZERO);
4279 printf("del hw filter failed to get memory\n");
4283 SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
4284 if (f->flags & IXL_FILTER_DEL) {
4285 e = &d[j]; // a pox on fvl long names :)
4286 bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
4287 e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
4288 e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
4289 /* delete entry from vsi list */
4290 SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
4298 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
4299 /* NOTE: returns ENOENT every time but seems to work fine,
4300 so we'll ignore that specific error. */
4301 // TODO: Does this still occur on current firmwares?
4302 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
4304 for (int i = 0; i < j; i++)
4305 sc += (!d[i].error_code);
4306 vsi->hw_filters_del += sc;
4308 "Failed to remove %d/%d filters, aq error %d\n",
4309 j - sc, j, hw->aq.asq_last_status);
4311 vsi->hw_filters_del += j;
4315 DEBUGOUT("ixl_del_hw_filters: end\n");
4320 ixl_enable_rings(struct ixl_vsi *vsi)
4322 struct ixl_pf *pf = vsi->back;
4323 struct i40e_hw *hw = &pf->hw;
4328 for (int i = 0; i < vsi->num_queues; i++) {
4329 index = vsi->first_queue + i;
4330 i40e_pre_tx_queue_cfg(hw, index, TRUE);
4332 reg = rd32(hw, I40E_QTX_ENA(index));
4333 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
4334 I40E_QTX_ENA_QENA_STAT_MASK;
4335 wr32(hw, I40E_QTX_ENA(index), reg);
4336 /* Verify the enable took */
4337 for (int j = 0; j < 10; j++) {
4338 reg = rd32(hw, I40E_QTX_ENA(index));
4339 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4341 i40e_msec_delay(10);
4343 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
4344 device_printf(pf->dev, "TX queue %d disabled!\n",
4349 reg = rd32(hw, I40E_QRX_ENA(index));
4350 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
4351 I40E_QRX_ENA_QENA_STAT_MASK;
4352 wr32(hw, I40E_QRX_ENA(index), reg);
4353 /* Verify the enable took */
4354 for (int j = 0; j < 10; j++) {
4355 reg = rd32(hw, I40E_QRX_ENA(index));
4356 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
4358 i40e_msec_delay(10);
4360 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
4361 device_printf(pf->dev, "RX queue %d disabled!\n",
4371 ixl_disable_rings(struct ixl_vsi *vsi)
4373 struct ixl_pf *pf = vsi->back;
4374 struct i40e_hw *hw = &pf->hw;
4379 for (int i = 0; i < vsi->num_queues; i++) {
4380 index = vsi->first_queue + i;
4382 i40e_pre_tx_queue_cfg(hw, index, FALSE);
4383 i40e_usec_delay(500);
4385 reg = rd32(hw, I40E_QTX_ENA(index));
4386 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4387 wr32(hw, I40E_QTX_ENA(index), reg);
4388 /* Verify the disable took */
4389 for (int j = 0; j < 10; j++) {
4390 reg = rd32(hw, I40E_QTX_ENA(index));
4391 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4393 i40e_msec_delay(10);
4395 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
4396 device_printf(pf->dev, "TX queue %d still enabled!\n",
4401 reg = rd32(hw, I40E_QRX_ENA(index));
4402 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4403 wr32(hw, I40E_QRX_ENA(index), reg);
4404 /* Verify the disable took */
4405 for (int j = 0; j < 10; j++) {
4406 reg = rd32(hw, I40E_QRX_ENA(index));
4407 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
4409 i40e_msec_delay(10);
4411 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
4412 device_printf(pf->dev, "RX queue %d still enabled!\n",
4422 * ixl_handle_mdd_event
4424 * Called from interrupt handler to identify possibly malicious vfs
4425 * (But also detects events from the PF, as well)
4428 ixl_handle_mdd_event(struct ixl_pf *pf)
4430 struct i40e_hw *hw = &pf->hw;
4431 device_t dev = pf->dev;
4432 bool mdd_detected = false;
4433 bool pf_mdd_detected = false;
4436 /* find what triggered the MDD event */
4437 reg = rd32(hw, I40E_GL_MDET_TX);
4438 if (reg & I40E_GL_MDET_TX_VALID_MASK) {
4439 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
4440 I40E_GL_MDET_TX_PF_NUM_SHIFT;
4441 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
4442 I40E_GL_MDET_TX_EVENT_SHIFT;
4443 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
4444 I40E_GL_MDET_TX_QUEUE_SHIFT;
4446 "Malicious Driver Detection event 0x%02x"
4447 " on TX queue %d pf number 0x%02x\n",
4448 event, queue, pf_num);
4449 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
4450 mdd_detected = true;
4452 reg = rd32(hw, I40E_GL_MDET_RX);
4453 if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4454 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
4455 I40E_GL_MDET_RX_FUNCTION_SHIFT;
4456 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
4457 I40E_GL_MDET_RX_EVENT_SHIFT;
4458 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
4459 I40E_GL_MDET_RX_QUEUE_SHIFT;
4461 "Malicious Driver Detection event 0x%02x"
4462 " on RX queue %d of function 0x%02x\n",
4463 event, queue, func);
4464 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4465 mdd_detected = true;
4469 reg = rd32(hw, I40E_PF_MDET_TX);
4470 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4471 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4473 "MDD TX event is for this function 0x%08x",
4475 pf_mdd_detected = true;
4477 reg = rd32(hw, I40E_PF_MDET_RX);
4478 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4479 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4481 "MDD RX event is for this function 0x%08x",
4483 pf_mdd_detected = true;
4487 /* re-enable mdd interrupt cause */
4488 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4489 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4490 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4495 ixl_enable_intr(struct ixl_vsi *vsi)
4497 struct i40e_hw *hw = vsi->hw;
4498 struct ixl_queue *que = vsi->queues;
4500 if (ixl_enable_msix) {
4501 for (int i = 0; i < vsi->num_queues; i++, que++)
4502 ixl_enable_queue(hw, que->me);
4504 ixl_enable_legacy(hw);
4508 ixl_disable_rings_intr(struct ixl_vsi *vsi)
4510 struct i40e_hw *hw = vsi->hw;
4511 struct ixl_queue *que = vsi->queues;
4513 for (int i = 0; i < vsi->num_queues; i++, que++)
4514 ixl_disable_queue(hw, que->me);
4518 ixl_disable_intr(struct ixl_vsi *vsi)
4520 struct i40e_hw *hw = vsi->hw;
4522 if (ixl_enable_msix)
4523 ixl_disable_adminq(hw);
4525 ixl_disable_legacy(hw);
4529 ixl_enable_adminq(struct i40e_hw *hw)
4533 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4534 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4535 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4536 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4541 ixl_disable_adminq(struct i40e_hw *hw)
4545 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4546 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4551 ixl_enable_queue(struct i40e_hw *hw, int id)
4555 reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4556 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4557 (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4558 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4562 ixl_disable_queue(struct i40e_hw *hw, int id)
4566 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4567 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4571 ixl_enable_legacy(struct i40e_hw *hw)
4574 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4575 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4576 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4577 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4581 ixl_disable_legacy(struct i40e_hw *hw)
4585 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4586 wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4590 ixl_update_stats_counters(struct ixl_pf *pf)
4592 struct i40e_hw *hw = &pf->hw;
4593 struct ixl_vsi *vsi = &pf->vsi;
4596 struct i40e_hw_port_stats *nsd = &pf->stats;
4597 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4599 /* Update hw stats */
4600 ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4601 pf->stat_offsets_loaded,
4602 &osd->crc_errors, &nsd->crc_errors);
4603 ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4604 pf->stat_offsets_loaded,
4605 &osd->illegal_bytes, &nsd->illegal_bytes);
4606 ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4607 I40E_GLPRT_GORCL(hw->port),
4608 pf->stat_offsets_loaded,
4609 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4610 ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4611 I40E_GLPRT_GOTCL(hw->port),
4612 pf->stat_offsets_loaded,
4613 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4614 ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4615 pf->stat_offsets_loaded,
4616 &osd->eth.rx_discards,
4617 &nsd->eth.rx_discards);
4618 ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4619 I40E_GLPRT_UPRCL(hw->port),
4620 pf->stat_offsets_loaded,
4621 &osd->eth.rx_unicast,
4622 &nsd->eth.rx_unicast);
4623 ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4624 I40E_GLPRT_UPTCL(hw->port),
4625 pf->stat_offsets_loaded,
4626 &osd->eth.tx_unicast,
4627 &nsd->eth.tx_unicast);
4628 ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4629 I40E_GLPRT_MPRCL(hw->port),
4630 pf->stat_offsets_loaded,
4631 &osd->eth.rx_multicast,
4632 &nsd->eth.rx_multicast);
4633 ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4634 I40E_GLPRT_MPTCL(hw->port),
4635 pf->stat_offsets_loaded,
4636 &osd->eth.tx_multicast,
4637 &nsd->eth.tx_multicast);
4638 ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4639 I40E_GLPRT_BPRCL(hw->port),
4640 pf->stat_offsets_loaded,
4641 &osd->eth.rx_broadcast,
4642 &nsd->eth.rx_broadcast);
4643 ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4644 I40E_GLPRT_BPTCL(hw->port),
4645 pf->stat_offsets_loaded,
4646 &osd->eth.tx_broadcast,
4647 &nsd->eth.tx_broadcast);
4649 ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4650 pf->stat_offsets_loaded,
4651 &osd->tx_dropped_link_down,
4652 &nsd->tx_dropped_link_down);
4653 ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4654 pf->stat_offsets_loaded,
4655 &osd->mac_local_faults,
4656 &nsd->mac_local_faults);
4657 ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4658 pf->stat_offsets_loaded,
4659 &osd->mac_remote_faults,
4660 &nsd->mac_remote_faults);
4661 ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4662 pf->stat_offsets_loaded,
4663 &osd->rx_length_errors,
4664 &nsd->rx_length_errors);
4666 /* Flow control (LFC) stats */
4667 ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4668 pf->stat_offsets_loaded,
4669 &osd->link_xon_rx, &nsd->link_xon_rx);
4670 ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4671 pf->stat_offsets_loaded,
4672 &osd->link_xon_tx, &nsd->link_xon_tx);
4673 ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4674 pf->stat_offsets_loaded,
4675 &osd->link_xoff_rx, &nsd->link_xoff_rx);
4676 ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4677 pf->stat_offsets_loaded,
4678 &osd->link_xoff_tx, &nsd->link_xoff_tx);
4680 /* Packet size stats rx */
4681 ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4682 I40E_GLPRT_PRC64L(hw->port),
4683 pf->stat_offsets_loaded,
4684 &osd->rx_size_64, &nsd->rx_size_64);
4685 ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4686 I40E_GLPRT_PRC127L(hw->port),
4687 pf->stat_offsets_loaded,
4688 &osd->rx_size_127, &nsd->rx_size_127);
4689 ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4690 I40E_GLPRT_PRC255L(hw->port),
4691 pf->stat_offsets_loaded,
4692 &osd->rx_size_255, &nsd->rx_size_255);
4693 ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4694 I40E_GLPRT_PRC511L(hw->port),
4695 pf->stat_offsets_loaded,
4696 &osd->rx_size_511, &nsd->rx_size_511);
4697 ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4698 I40E_GLPRT_PRC1023L(hw->port),
4699 pf->stat_offsets_loaded,
4700 &osd->rx_size_1023, &nsd->rx_size_1023);
4701 ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4702 I40E_GLPRT_PRC1522L(hw->port),
4703 pf->stat_offsets_loaded,
4704 &osd->rx_size_1522, &nsd->rx_size_1522);
4705 ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4706 I40E_GLPRT_PRC9522L(hw->port),
4707 pf->stat_offsets_loaded,
4708 &osd->rx_size_big, &nsd->rx_size_big);
4710 /* Packet size stats tx */
4711 ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4712 I40E_GLPRT_PTC64L(hw->port),
4713 pf->stat_offsets_loaded,
4714 &osd->tx_size_64, &nsd->tx_size_64);
4715 ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4716 I40E_GLPRT_PTC127L(hw->port),
4717 pf->stat_offsets_loaded,
4718 &osd->tx_size_127, &nsd->tx_size_127);
4719 ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4720 I40E_GLPRT_PTC255L(hw->port),
4721 pf->stat_offsets_loaded,
4722 &osd->tx_size_255, &nsd->tx_size_255);
4723 ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4724 I40E_GLPRT_PTC511L(hw->port),
4725 pf->stat_offsets_loaded,
4726 &osd->tx_size_511, &nsd->tx_size_511);
4727 ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4728 I40E_GLPRT_PTC1023L(hw->port),
4729 pf->stat_offsets_loaded,
4730 &osd->tx_size_1023, &nsd->tx_size_1023);
4731 ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4732 I40E_GLPRT_PTC1522L(hw->port),
4733 pf->stat_offsets_loaded,
4734 &osd->tx_size_1522, &nsd->tx_size_1522);
4735 ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4736 I40E_GLPRT_PTC9522L(hw->port),
4737 pf->stat_offsets_loaded,
4738 &osd->tx_size_big, &nsd->tx_size_big);
4740 ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4741 pf->stat_offsets_loaded,
4742 &osd->rx_undersize, &nsd->rx_undersize);
4743 ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4744 pf->stat_offsets_loaded,
4745 &osd->rx_fragments, &nsd->rx_fragments);
4746 ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4747 pf->stat_offsets_loaded,
4748 &osd->rx_oversize, &nsd->rx_oversize);
4749 ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4750 pf->stat_offsets_loaded,
4751 &osd->rx_jabber, &nsd->rx_jabber);
4752 pf->stat_offsets_loaded = true;
4755 /* Update vsi stats */
4756 ixl_update_vsi_stats(vsi);
4758 for (int i = 0; i < pf->num_vfs; i++) {
4760 if (vf->vf_flags & VF_FLAG_ENABLED)
4761 ixl_update_eth_stats(&pf->vfs[i].vsi);
4766 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf)
4768 struct i40e_hw *hw = &pf->hw;
4769 struct ixl_vsi *vsi = &pf->vsi;
4770 device_t dev = pf->dev;
4774 is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING);
4779 error = i40e_shutdown_lan_hmc(hw);
4782 "Shutdown LAN HMC failed with code %d\n", error);
4783 ixl_disable_adminq(hw);
4784 ixl_teardown_adminq_msix(pf);
4785 error = i40e_shutdown_adminq(hw);
4788 "Shutdown Admin queue failed with code %d\n", error);
4791 error = i40e_init_adminq(hw);
4792 if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
4793 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
4796 error = ixl_setup_adminq_msix(pf);
4798 device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
4801 ixl_configure_intr0_msix(pf);
4802 ixl_enable_adminq(hw);
4803 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4804 hw->func_caps.num_rx_qp, 0, 0);
4806 device_printf(dev, "init_lan_hmc failed: %d\n", error);
4808 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4810 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
4819 ixl_handle_empr_reset(struct ixl_pf *pf)
4821 struct i40e_hw *hw = &pf->hw;
4822 device_t dev = pf->dev;
4826 /* Typically finishes within 3-4 seconds */
4827 while (count++ < 100) {
4828 reg = rd32(hw, I40E_GLGEN_RSTAT)
4829 & I40E_GLGEN_RSTAT_DEVSTATE_MASK;
4831 i40e_msec_delay(100);
4837 device_printf(dev, "EMPR reset wait count: %d\n", count);
4840 device_printf(dev, "Rebuilding driver state...\n");
4841 ixl_rebuild_hw_structs_after_reset(pf);
4842 device_printf(dev, "Rebuilding driver state done.\n");
4844 atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
4848 ** Tasklet handler for MSIX Adminq interrupts
4849 ** - do outside interrupt since it might sleep
4852 ixl_do_adminq(void *context, int pending)
4854 struct ixl_pf *pf = context;
4855 struct i40e_hw *hw = &pf->hw;
4856 struct i40e_arq_event_info event;
4858 device_t dev = pf->dev;
4862 if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
4863 /* Flag cleared at end of this function */
4864 ixl_handle_empr_reset(pf);
4868 /* Admin Queue handling */
4869 event.buf_len = IXL_AQ_BUF_SZ;
4870 event.msg_buf = malloc(event.buf_len,
4871 M_DEVBUF, M_NOWAIT | M_ZERO);
4872 if (!event.msg_buf) {
4873 device_printf(dev, "%s: Unable to allocate memory for Admin"
4874 " Queue event!\n", __func__);
4879 /* clean and process any events */
4881 ret = i40e_clean_arq_element(hw, &event, &result);
4884 opcode = LE16_TO_CPU(event.desc.opcode);
4886 device_printf(dev, "%s: Admin Queue event: %#06x\n", __func__,
4890 case i40e_aqc_opc_get_link_status:
4891 ixl_link_event(pf, &event);
4893 case i40e_aqc_opc_send_msg_to_pf:
4895 ixl_handle_vf_msg(pf, &event);
4898 case i40e_aqc_opc_event_lan_overflow:
4903 } while (result && (loop++ < IXL_ADM_LIMIT));
4905 free(event.msg_buf, M_DEVBUF);
4908 * If there are still messages to process, reschedule ourselves.
4909 * Otherwise, re-enable our interrupt and go to sleep.
4912 taskqueue_enqueue(pf->tq, &pf->adminq);
4914 ixl_enable_adminq(hw);
4920 * Update VSI-specific ethernet statistics counters.
4923 ixl_update_eth_stats(struct ixl_vsi *vsi)
4925 struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4926 struct i40e_hw *hw = &pf->hw;
4927 struct i40e_eth_stats *es;
4928 struct i40e_eth_stats *oes;
4929 struct i40e_hw_port_stats *nsd;
4930 u16 stat_idx = vsi->info.stat_counter_idx;
4932 es = &vsi->eth_stats;
4933 oes = &vsi->eth_stats_offsets;
4936 /* Gather up the stats that the hw collects */
4937 ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4938 vsi->stat_offsets_loaded,
4939 &oes->tx_errors, &es->tx_errors);
4940 ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4941 vsi->stat_offsets_loaded,
4942 &oes->rx_discards, &es->rx_discards);
4944 ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4945 I40E_GLV_GORCL(stat_idx),
4946 vsi->stat_offsets_loaded,
4947 &oes->rx_bytes, &es->rx_bytes);
4948 ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4949 I40E_GLV_UPRCL(stat_idx),
4950 vsi->stat_offsets_loaded,
4951 &oes->rx_unicast, &es->rx_unicast);
4952 ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4953 I40E_GLV_MPRCL(stat_idx),
4954 vsi->stat_offsets_loaded,
4955 &oes->rx_multicast, &es->rx_multicast);
4956 ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4957 I40E_GLV_BPRCL(stat_idx),
4958 vsi->stat_offsets_loaded,
4959 &oes->rx_broadcast, &es->rx_broadcast);
4961 ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4962 I40E_GLV_GOTCL(stat_idx),
4963 vsi->stat_offsets_loaded,
4964 &oes->tx_bytes, &es->tx_bytes);
4965 ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4966 I40E_GLV_UPTCL(stat_idx),
4967 vsi->stat_offsets_loaded,
4968 &oes->tx_unicast, &es->tx_unicast);
4969 ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4970 I40E_GLV_MPTCL(stat_idx),
4971 vsi->stat_offsets_loaded,
4972 &oes->tx_multicast, &es->tx_multicast);
4973 ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4974 I40E_GLV_BPTCL(stat_idx),
4975 vsi->stat_offsets_loaded,
4976 &oes->tx_broadcast, &es->tx_broadcast);
4977 vsi->stat_offsets_loaded = true;
4981 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4985 struct i40e_eth_stats *es;
4988 struct i40e_hw_port_stats *nsd;
4992 es = &vsi->eth_stats;
4995 ixl_update_eth_stats(vsi);
4997 tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4998 for (int i = 0; i < vsi->num_queues; i++)
4999 tx_discards += vsi->queues[i].txr.br->br_drops;
5001 /* Update ifnet stats */
5002 IXL_SET_IPACKETS(vsi, es->rx_unicast +
5005 IXL_SET_OPACKETS(vsi, es->tx_unicast +
5008 IXL_SET_IBYTES(vsi, es->rx_bytes);
5009 IXL_SET_OBYTES(vsi, es->tx_bytes);
5010 IXL_SET_IMCASTS(vsi, es->rx_multicast);
5011 IXL_SET_OMCASTS(vsi, es->tx_multicast);
5013 IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
5014 nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
5016 IXL_SET_OERRORS(vsi, es->tx_errors);
5017 IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
5018 IXL_SET_OQDROPS(vsi, tx_discards);
5019 IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
5020 IXL_SET_COLLISIONS(vsi, 0);
5024 * Reset all of the stats for the given pf
5026 void ixl_pf_reset_stats(struct ixl_pf *pf)
5028 bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
5029 bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
5030 pf->stat_offsets_loaded = false;
5034 * Resets all stats of the given vsi
5036 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
5038 bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
5039 bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
5040 vsi->stat_offsets_loaded = false;
5044 * Read and update a 48 bit stat from the hw
5046 * Since the device stats are not reset at PFReset, they likely will not
5047 * be zeroed when the driver starts. We'll save the first values read
5048 * and use them as offsets to be subtracted from the raw values in order
5049 * to report stats that count from zero.
5052 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
5053 bool offset_loaded, u64 *offset, u64 *stat)
5057 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
5058 new_data = rd64(hw, loreg);
5061 * Use two rd32's instead of one rd64; FreeBSD versions before
5062 * 10 don't support 8 byte bus reads/writes.
5064 new_data = rd32(hw, loreg);
5065 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
5070 if (new_data >= *offset)
5071 *stat = new_data - *offset;
5073 *stat = (new_data + ((u64)1 << 48)) - *offset;
5074 *stat &= 0xFFFFFFFFFFFFULL;
5078 * Read and update a 32 bit stat from the hw
5081 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
5082 bool offset_loaded, u64 *offset, u64 *stat)
5086 new_data = rd32(hw, reg);
5089 if (new_data >= *offset)
5090 *stat = (u32)(new_data - *offset);
5092 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
5096 ixl_add_device_sysctls(struct ixl_pf *pf)
5098 device_t dev = pf->dev;
5100 /* Set up sysctls */
5101 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5102 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5103 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
5104 pf, 0, ixl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC);
5106 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5107 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5108 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
5109 pf, 0, ixl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE);
5111 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5112 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5113 OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
5114 pf, 0, ixl_current_speed, "A", "Current Port Speed");
5116 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5117 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5118 OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
5119 pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
5122 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5123 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5124 OID_AUTO, "rx_itr", CTLFLAG_RW,
5125 &ixl_rx_itr, 0, "RX ITR");
5127 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5128 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5129 OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
5130 &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
5132 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5133 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5134 OID_AUTO, "tx_itr", CTLFLAG_RW,
5135 &ixl_tx_itr, 0, "TX ITR");
5137 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5138 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5139 OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
5140 &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
5143 #ifdef IXL_DEBUG_SYSCTL
5144 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5145 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5146 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
5147 ixl_debug_info, "I", "Debug Information");
5149 /* Shared-code debug message level */
5150 SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5151 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5152 OID_AUTO, "debug_mask", CTLFLAG_RW,
5153 &pf->hw.debug_mask, 0, "Debug Message Level");
5155 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5156 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5157 OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
5158 pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS);
5160 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5161 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5162 OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
5163 pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
5165 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5166 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5167 OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
5168 pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
5170 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5171 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5172 OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
5173 pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
5175 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5176 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5177 OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
5178 pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
5181 SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5182 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5183 OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
5184 0, "PF/VF Virtual Channel debug level");
5190 ** Set flow control using sysctl:
5197 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
5200 * TODO: ensure tx CRC by hardware should be enabled
5201 * if tx flow control is enabled.
5202 * ^ N/A for 40G ports
5204 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5205 struct i40e_hw *hw = &pf->hw;
5206 device_t dev = pf->dev;
5207 int requested_fc, error = 0;
5208 enum i40e_status_code aq_error = 0;
5212 requested_fc = pf->fc;
5213 error = sysctl_handle_int(oidp, &requested_fc, 0, req);
5214 if ((error) || (req->newptr == NULL))
5216 if (requested_fc < 0 || requested_fc > 3) {
5218 "Invalid fc mode; valid modes are 0 through 3\n");
5222 /* Set fc ability for port */
5223 hw->fc.requested_mode = requested_fc;
5224 aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
5227 "%s: Error setting new fc mode %d; fc_err %#x\n",
5228 __func__, aq_error, fc_aq_err);
5231 pf->fc = requested_fc;
5233 /* Get new link state */
5234 i40e_msec_delay(250);
5235 hw->phy.get_link_info = TRUE;
5236 i40e_get_link_status(hw, &pf->link_up);
5242 ixl_current_speed(SYSCTL_HANDLER_ARGS)
5244 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5245 struct i40e_hw *hw = &pf->hw;
5246 int error = 0, index = 0;
5257 ixl_update_link_status(pf);
5259 switch (hw->phy.link_info.link_speed) {
5260 case I40E_LINK_SPEED_100MB:
5263 case I40E_LINK_SPEED_1GB:
5266 case I40E_LINK_SPEED_10GB:
5269 case I40E_LINK_SPEED_40GB:
5272 case I40E_LINK_SPEED_20GB:
5275 case I40E_LINK_SPEED_UNKNOWN:
5281 error = sysctl_handle_string(oidp, speeds[index],
5282 strlen(speeds[index]), req);
5287 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
5289 struct i40e_hw *hw = &pf->hw;
5290 device_t dev = pf->dev;
5291 struct i40e_aq_get_phy_abilities_resp abilities;
5292 struct i40e_aq_set_phy_config config;
5293 enum i40e_status_code aq_error = 0;
5295 /* Get current capability information */
5296 aq_error = i40e_aq_get_phy_capabilities(hw,
5297 FALSE, FALSE, &abilities, NULL);
5300 "%s: Error getting phy capabilities %d,"
5301 " aq error: %d\n", __func__, aq_error,
5302 hw->aq.asq_last_status);
5306 /* Prepare new config */
5307 bzero(&config, sizeof(config));
5308 config.phy_type = abilities.phy_type;
5309 config.abilities = abilities.abilities
5310 | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
5311 config.eee_capability = abilities.eee_capability;
5312 config.eeer = abilities.eeer_val;
5313 config.low_power_ctrl = abilities.d3_lpan;
5314 /* Translate into aq cmd link_speed */
5316 config.link_speed |= I40E_LINK_SPEED_40GB;
5318 config.link_speed |= I40E_LINK_SPEED_20GB;
5320 config.link_speed |= I40E_LINK_SPEED_10GB;
5322 config.link_speed |= I40E_LINK_SPEED_1GB;
5324 config.link_speed |= I40E_LINK_SPEED_100MB;
5326 /* Do aq command & restart link */
5327 aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
5330 "%s: Error setting new phy config %d,"
5331 " aq error: %d\n", __func__, aq_error,
5332 hw->aq.asq_last_status);
5337 ** This seems a bit heavy handed, but we
5338 ** need to get a reinit on some devices
5341 ixl_stop_locked(pf);
5342 ixl_init_locked(pf);
5349 ** Control link advertise speed:
5351 ** 0x1 - advertise 100 Mb
5352 ** 0x2 - advertise 1G
5353 ** 0x4 - advertise 10G
5354 ** 0x8 - advertise 20G
5355 ** 0x10 - advertise 40G
5357 ** Set to 0 to disable link
5360 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
5362 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5363 struct i40e_hw *hw = &pf->hw;
5364 device_t dev = pf->dev;
5365 int requested_ls = 0;
5368 /* Read in new mode */
5369 requested_ls = pf->advertised_speed;
5370 error = sysctl_handle_int(oidp, &requested_ls, 0, req);
5371 if ((error) || (req->newptr == NULL))
5373 /* Check for sane value */
5374 if (requested_ls > 0x10) {
5375 device_printf(dev, "Invalid advertised speed; "
5376 "valid modes are 0x1 through 0x10\n");
5379 /* Then check for validity based on adapter type */
5380 switch (hw->device_id) {
5381 case I40E_DEV_ID_10G_BASE_T:
5382 case I40E_DEV_ID_10G_BASE_T4:
5384 if (requested_ls & ~(0x7)) {
5386 "Only 100M/1G/10G speeds supported on this device.\n");
5390 case I40E_DEV_ID_20G_KR2:
5391 case I40E_DEV_ID_20G_KR2_A:
5393 if (requested_ls & ~(0xE)) {
5395 "Only 1G/10G/20G speeds supported on this device.\n");
5399 case I40E_DEV_ID_KX_B:
5400 case I40E_DEV_ID_QSFP_A:
5401 case I40E_DEV_ID_QSFP_B:
5403 if (requested_ls & ~(0x10)) {
5405 "Only 40G speeds supported on this device.\n");
5411 if (requested_ls & ~(0x6)) {
5413 "Only 1/10Gbs speeds are supported on this device.\n");
5419 /* Exit if no change */
5420 if (pf->advertised_speed == requested_ls)
5423 error = ixl_set_advertised_speeds(pf, requested_ls);
5427 pf->advertised_speed = requested_ls;
5428 ixl_update_link_status(pf);
5433 ** Get the width and transaction speed of
5434 ** the bus this adapter is plugged into.
5437 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
5442 /* Get the PCI Express Capabilities offset */
5443 pci_find_cap(dev, PCIY_EXPRESS, &offset);
5445 /* ...and read the Link Status Register */
5446 link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
5448 switch (link & I40E_PCI_LINK_WIDTH) {
5449 case I40E_PCI_LINK_WIDTH_1:
5450 hw->bus.width = i40e_bus_width_pcie_x1;
5452 case I40E_PCI_LINK_WIDTH_2:
5453 hw->bus.width = i40e_bus_width_pcie_x2;
5455 case I40E_PCI_LINK_WIDTH_4:
5456 hw->bus.width = i40e_bus_width_pcie_x4;
5458 case I40E_PCI_LINK_WIDTH_8:
5459 hw->bus.width = i40e_bus_width_pcie_x8;
5462 hw->bus.width = i40e_bus_width_unknown;
5466 switch (link & I40E_PCI_LINK_SPEED) {
5467 case I40E_PCI_LINK_SPEED_2500:
5468 hw->bus.speed = i40e_bus_speed_2500;
5470 case I40E_PCI_LINK_SPEED_5000:
5471 hw->bus.speed = i40e_bus_speed_5000;
5473 case I40E_PCI_LINK_SPEED_8000:
5474 hw->bus.speed = i40e_bus_speed_8000;
5477 hw->bus.speed = i40e_bus_speed_unknown;
5481 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
5482 ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
5483 (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
5484 (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
5485 (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
5486 (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
5487 (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
5490 if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
5491 (hw->bus.speed < i40e_bus_speed_8000)) {
5492 device_printf(dev, "PCI-Express bandwidth available"
5493 " for this device\n may be insufficient for"
5494 " optimal performance.\n");
5495 device_printf(dev, "For expected performance a x8 "
5496 "PCIE Gen3 slot is required.\n");
5503 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
5505 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5506 struct i40e_hw *hw = &pf->hw;
5509 sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5510 ixl_nvm_version_str(hw, sbuf);
5519 ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma)
5521 if ((nvma->command == I40E_NVM_READ) &&
5522 ((nvma->config & 0xFF) == 0xF) &&
5523 (((nvma->config & 0xF00) >> 8) == 0xF) &&
5524 (nvma->offset == 0) &&
5525 (nvma->data_size == 1)) {
5526 // device_printf(dev, "- Get Driver Status Command\n");
5528 else if (nvma->command == I40E_NVM_READ) {
5532 switch (nvma->command) {
5534 device_printf(dev, "- command: I40E_NVM_READ\n");
5537 device_printf(dev, "- command: I40E_NVM_WRITE\n");
5540 device_printf(dev, "- command: unknown 0x%08x\n", nvma->command);
5544 device_printf(dev, "- config (ptr) : 0x%02x\n", nvma->config & 0xFF);
5545 device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8);
5546 device_printf(dev, "- offset : 0x%08x\n", nvma->offset);
5547 device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size);
5553 ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd)
5555 struct i40e_hw *hw = &pf->hw;
5556 struct i40e_nvm_access *nvma;
5557 device_t dev = pf->dev;
5558 enum i40e_status_code status = 0;
5561 DEBUGFUNC("ixl_handle_nvmupd_cmd");
5564 if (ifd->ifd_len < sizeof(struct i40e_nvm_access) ||
5565 ifd->ifd_data == NULL) {
5566 device_printf(dev, "%s: incorrect ifdrv length or data pointer\n",
5568 device_printf(dev, "%s: ifdrv length: %lu, sizeof(struct i40e_nvm_access): %lu\n",
5569 __func__, ifd->ifd_len, sizeof(struct i40e_nvm_access));
5570 device_printf(dev, "%s: data pointer: %p\n", __func__,
5575 nvma = (struct i40e_nvm_access *)ifd->ifd_data;
5578 ixl_print_nvm_cmd(dev, nvma);
5581 if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
5583 while (count++ < 100) {
5584 i40e_msec_delay(100);
5585 if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING))
5590 if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) {
5592 status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno);
5599 device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n",
5603 * -EPERM is actually ERESTART, which the kernel interprets as it needing
5604 * to run this ioctl again. So use -EACCES for -EPERM instead.
5606 if (perrno == -EPERM)
5612 #ifdef IXL_DEBUG_SYSCTL
5614 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
5616 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5617 struct i40e_hw *hw = &pf->hw;
5618 struct i40e_link_status link_status;
5621 enum i40e_status_code aq_error = 0;
5623 aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
5625 printf("i40e_aq_get_link_info() error %d\n", aq_error);
5630 "PHY Type : %#04x\n"
5632 "Link info: %#04x\n"
5634 "Ext info : %#04x\n"
5638 link_status.phy_type, link_status.link_speed,
5639 link_status.link_info, link_status.an_info,
5640 link_status.ext_info, link_status.max_frame_size,
5641 link_status.pacing, link_status.crc_enable);
5643 return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5647 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
5649 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5650 struct i40e_hw *hw = &pf->hw;
5652 enum i40e_status_code aq_error = 0;
5654 struct i40e_aq_get_phy_abilities_resp abilities;
5656 aq_error = i40e_aq_get_phy_capabilities(hw,
5657 TRUE, FALSE, &abilities, NULL);
5659 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
5664 "PHY Type : %#010x\n"
5666 "Abilities: %#04x\n"
5668 "EEER reg : %#010x\n"
5670 abilities.phy_type, abilities.link_speed,
5671 abilities.abilities, abilities.eee_capability,
5672 abilities.eeer_val, abilities.d3_lpan);
5674 return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5678 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5680 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5681 struct ixl_vsi *vsi = &pf->vsi;
5682 struct ixl_mac_filter *f;
5687 int ftl_counter = 0;
5691 SLIST_FOREACH(f, &vsi->ftl, next) {
5696 sysctl_handle_string(oidp, "(none)", 6, req);
5700 buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5701 buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5703 sprintf(buf_i++, "\n");
5704 SLIST_FOREACH(f, &vsi->ftl, next) {
5706 MAC_FORMAT ", vlan %4d, flags %#06x",
5707 MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5709 /* don't print '\n' for last entry */
5710 if (++ftl_counter != ftl_len) {
5711 sprintf(buf_i, "\n");
5716 error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5718 printf("sysctl error: %d\n", error);
5719 free(buf, M_DEVBUF);
5723 #define IXL_SW_RES_SIZE 0x14
5725 ixl_res_alloc_cmp(const void *a, const void *b)
5727 const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5728 one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5729 two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5731 return ((int)one->resource_type - (int)two->resource_type);
5735 * Longest string length: 25
5738 ixl_switch_res_type_string(u8 type)
5740 static char * ixl_switch_res_type_strings[0x14] = {
5743 "Perfect Match MAC address",
5746 "Multicast hash entry",
5747 "Unicast hash entry",
5751 "VLAN Statistic Pool",
5754 "Inner VLAN Forward filter",
5764 return ixl_switch_res_type_strings[type];
5766 return "(Reserved)";
5770 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5772 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5773 struct i40e_hw *hw = &pf->hw;
5774 device_t dev = pf->dev;
5779 struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5781 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5783 device_printf(dev, "Could not allocate sbuf for output.\n");
5787 bzero(resp, sizeof(resp));
5788 error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5794 "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5795 __func__, error, hw->aq.asq_last_status);
5800 /* Sort entries by type for display */
5801 qsort(resp, num_entries,
5802 sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5803 &ixl_res_alloc_cmp);
5805 sbuf_cat(buf, "\n");
5806 sbuf_printf(buf, "# of entries: %d\n", num_entries);
5808 " Type | Guaranteed | Total | Used | Un-allocated\n"
5809 " | (this) | (all) | (this) | (all) \n");
5810 for (int i = 0; i < num_entries; i++) {
5812 "%25s | %10d %5d %6d %12d",
5813 ixl_switch_res_type_string(resp[i].resource_type),
5817 resp[i].total_unalloced);
5818 if (i < num_entries - 1)
5819 sbuf_cat(buf, "\n");
5822 error = sbuf_finish(buf);
5824 device_printf(dev, "Error finishing sbuf: %d\n", error);
5831 ** Caller must init and delete sbuf; this function will clear and
5832 ** finish it for caller.
5834 ** XXX: Cannot use the SEID for this, since there is no longer a
5835 ** fixed mapping between SEID and element type.
5838 ixl_switch_element_string(struct sbuf *s,
5839 struct i40e_aqc_switch_config_element_resp *element)
5843 switch (element->element_type) {
5844 case I40E_AQ_SW_ELEM_TYPE_MAC:
5845 sbuf_printf(s, "MAC %3d", element->element_info);
5847 case I40E_AQ_SW_ELEM_TYPE_PF:
5848 sbuf_printf(s, "PF %3d", element->element_info);
5850 case I40E_AQ_SW_ELEM_TYPE_VF:
5851 sbuf_printf(s, "VF %3d", element->element_info);
5853 case I40E_AQ_SW_ELEM_TYPE_EMP:
5856 case I40E_AQ_SW_ELEM_TYPE_BMC:
5859 case I40E_AQ_SW_ELEM_TYPE_PV:
5862 case I40E_AQ_SW_ELEM_TYPE_VEB:
5865 case I40E_AQ_SW_ELEM_TYPE_PA:
5868 case I40E_AQ_SW_ELEM_TYPE_VSI:
5869 sbuf_printf(s, "VSI %3d", element->element_info);
5877 return sbuf_data(s);
5881 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5883 struct ixl_pf *pf = (struct ixl_pf *)arg1;
5884 struct i40e_hw *hw = &pf->hw;
5885 device_t dev = pf->dev;
5890 u8 aq_buf[I40E_AQ_LARGE_BUF];
5892 struct i40e_aqc_get_switch_config_resp *sw_config;
5893 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5895 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5897 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5901 error = i40e_aq_get_switch_config(hw, sw_config,
5902 sizeof(aq_buf), &next, NULL);
5905 "%s: aq_get_switch_config() error %d, aq error %d\n",
5906 __func__, error, hw->aq.asq_last_status);
5911 device_printf(dev, "%s: TODO: get more config with SEID %d\n",
5914 nmbuf = sbuf_new_auto();
5916 device_printf(dev, "Could not allocate sbuf for name output.\n");
5921 sbuf_cat(buf, "\n");
5922 // Assuming <= 255 elements in switch
5923 sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported);
5924 sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total);
5926 ** Revision -- all elements are revision 1 for now
5929 "SEID ( Name ) | Uplink | Downlink | Conn Type\n"
5930 " | | | (uplink)\n");
5931 for (int i = 0; i < sw_config->header.num_reported; i++) {
5932 // "%4d (%8s) | %8s %8s %#8x",
5933 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5935 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5936 &sw_config->element[i]));
5937 sbuf_cat(buf, " | ");
5938 sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid);
5940 sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid);
5942 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5943 if (i < sw_config->header.num_reported - 1)
5944 sbuf_cat(buf, "\n");
5948 error = sbuf_finish(buf);
5950 device_printf(dev, "Error finishing sbuf: %d\n", error);
5958 ixl_debug_info(SYSCTL_HANDLER_ARGS)
5961 int error, input = 0;
5963 error = sysctl_handle_int(oidp, &input, 0, req);
5965 if (error || !req->newptr)
5969 pf = (struct ixl_pf *)arg1;
5970 ixl_print_debug_info(pf);
5977 ixl_print_debug_info(struct ixl_pf *pf)
5979 struct i40e_hw *hw = &pf->hw;
5980 struct ixl_vsi *vsi = &pf->vsi;
5981 struct ixl_queue *que = vsi->queues;
5982 struct rx_ring *rxr = &que->rxr;
5983 struct tx_ring *txr = &que->txr;
5987 printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
5988 printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
5989 printf("RX next check = %x\n", rxr->next_check);
5990 printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
5991 printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
5992 printf("TX desc avail = %x\n", txr->avail);
5994 reg = rd32(hw, I40E_GLV_GORCL(0xc));
5995 printf("RX Bytes = %x\n", reg);
5996 reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
5997 printf("Port RX Bytes = %x\n", reg);
5998 reg = rd32(hw, I40E_GLV_RDPC(0xc));
5999 printf("RX discard = %x\n", reg);
6000 reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
6001 printf("Port RX discard = %x\n", reg);
6003 reg = rd32(hw, I40E_GLV_TEPC(0xc));
6004 printf("TX errors = %x\n", reg);
6005 reg = rd32(hw, I40E_GLV_GOTCL(0xc));
6006 printf("TX Bytes = %x\n", reg);
6008 reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
6009 printf("RX undersize = %x\n", reg);
6010 reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
6011 printf("RX fragments = %x\n", reg);
6012 reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
6013 printf("RX oversize = %x\n", reg);
6014 reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
6015 printf("RX length error = %x\n", reg);
6016 reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
6017 printf("mac remote fault = %x\n", reg);
6018 reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
6019 printf("mac local fault = %x\n", reg);
6022 #endif /* IXL_DEBUG_SYSCTL */
6026 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6029 struct ixl_vsi *vsi;
6030 struct i40e_vsi_context vsi_ctx;
6032 uint16_t first_queue;
6033 enum i40e_status_code code;
6038 vsi_ctx.pf_num = hw->pf_id;
6039 vsi_ctx.uplink_seid = pf->veb_seid;
6040 vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
6041 vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6042 vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
6044 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6046 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6047 vsi_ctx.info.switch_id = htole16(0);
6049 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
6050 vsi_ctx.info.sec_flags = 0;
6051 if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
6052 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
6054 /* TODO: If a port VLAN is set, then this needs to be changed */
6055 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6056 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6057 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
6059 vsi_ctx.info.valid_sections |=
6060 htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
6061 vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
6062 first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
6063 for (i = 0; i < IXLV_MAX_QUEUES; i++)
6064 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
6065 for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
6066 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
6068 vsi_ctx.info.tc_mapping[0] = htole16(
6069 (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6070 (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
6072 code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
6073 if (code != I40E_SUCCESS)
6074 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6075 vf->vsi.seid = vsi_ctx.seid;
6076 vf->vsi.vsi_num = vsi_ctx.vsi_number;
6077 vf->vsi.first_queue = first_queue;
6078 vf->vsi.num_queues = IXLV_MAX_QUEUES;
6080 code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
6081 if (code != I40E_SUCCESS)
6082 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6084 code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
6085 if (code != I40E_SUCCESS) {
6086 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
6087 ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6088 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6091 memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
6096 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6103 error = ixl_vf_alloc_vsi(pf, vf);
6107 vf->vsi.hw_filters_add = 0;
6108 vf->vsi.hw_filters_del = 0;
6109 ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
6110 ixl_reconfigure_filters(&vf->vsi);
6116 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
6123 * Two queues are mapped in a single register, so we have to do some
6124 * gymnastics to convert the queue number into a register index and
6128 shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
6130 qtable = i40e_read_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
6131 qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
6132 qtable |= val << shift;
6133 i40e_write_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
6137 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
6146 * Contiguous mappings aren't actually supported by the hardware,
6147 * so we have to use non-contiguous mappings.
6149 i40e_write_rx_ctl(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
6150 I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
6152 wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
6153 I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
6155 for (i = 0; i < vf->vsi.num_queues; i++) {
6156 qtable = (vf->vsi.first_queue + i) <<
6157 I40E_VPLAN_QTABLE_QINDEX_SHIFT;
6159 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
6162 /* Map queues allocated to VF to its VSI. */
6163 for (i = 0; i < vf->vsi.num_queues; i++)
6164 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
6166 /* Set rest of VSI queues as unused. */
6167 for (; i < IXL_MAX_VSI_QUEUES; i++)
6168 ixl_vf_map_vsi_queue(hw, vf, i,
6169 I40E_VSILAN_QTABLE_QINDEX_0_MASK);
6175 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
6184 i40e_aq_delete_element(hw, vsi->seid, NULL);
6188 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
6191 wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
6196 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
6199 wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
6200 I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
6205 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
6208 uint32_t vfint_reg, vpint_reg;
6213 ixl_vf_vsi_release(pf, &vf->vsi);
6215 /* Index 0 has a special register. */
6216 ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
6218 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6219 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
6220 ixl_vf_disable_queue_intr(hw, vfint_reg);
6223 /* Index 0 has a special register. */
6224 ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
6226 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6227 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
6228 ixl_vf_unregister_intr(hw, vpint_reg);
6231 vf->vsi.num_queues = 0;
6235 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
6239 uint16_t global_vf_num;
6243 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6245 wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
6246 (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
6247 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6248 ciad = rd32(hw, I40E_PF_PCI_CIAD);
6249 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
6258 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6265 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6266 vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6267 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6270 ixl_reinit_vf(pf, vf);
6274 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6277 uint32_t vfrstat, vfrtrig;
6282 error = ixl_flush_pcie(pf, vf);
6284 device_printf(pf->dev,
6285 "Timed out waiting for PCIe activity to stop on VF-%d\n",
6288 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6291 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
6292 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
6296 if (i == IXL_VF_RESET_TIMEOUT)
6297 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
6299 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
6301 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6302 vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6303 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6305 if (vf->vsi.seid != 0)
6306 ixl_disable_rings(&vf->vsi);
6308 ixl_vf_release_resources(pf, vf);
6309 ixl_vf_setup_vsi(pf, vf);
6310 ixl_vf_map_queues(pf, vf);
6312 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
6317 ixl_vc_opcode_str(uint16_t op)
6321 case I40E_VIRTCHNL_OP_VERSION:
6323 case I40E_VIRTCHNL_OP_RESET_VF:
6324 return ("RESET_VF");
6325 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6326 return ("GET_VF_RESOURCES");
6327 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6328 return ("CONFIG_TX_QUEUE");
6329 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6330 return ("CONFIG_RX_QUEUE");
6331 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6332 return ("CONFIG_VSI_QUEUES");
6333 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6334 return ("CONFIG_IRQ_MAP");
6335 case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6336 return ("ENABLE_QUEUES");
6337 case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6338 return ("DISABLE_QUEUES");
6339 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6340 return ("ADD_ETHER_ADDRESS");
6341 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6342 return ("DEL_ETHER_ADDRESS");
6343 case I40E_VIRTCHNL_OP_ADD_VLAN:
6344 return ("ADD_VLAN");
6345 case I40E_VIRTCHNL_OP_DEL_VLAN:
6346 return ("DEL_VLAN");
6347 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6348 return ("CONFIG_PROMISCUOUS_MODE");
6349 case I40E_VIRTCHNL_OP_GET_STATS:
6350 return ("GET_STATS");
6351 case I40E_VIRTCHNL_OP_FCOE:
6353 case I40E_VIRTCHNL_OP_EVENT:
6361 ixl_vc_opcode_level(uint16_t opcode)
6364 case I40E_VIRTCHNL_OP_GET_STATS:
6372 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6373 enum i40e_status_code status, void *msg, uint16_t len)
6379 global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
6381 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
6382 "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
6383 ixl_vc_opcode_str(op), op, status, vf->vf_num);
6385 i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
6389 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
6392 ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
6396 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6397 enum i40e_status_code status, const char *file, int line)
6400 I40E_VC_DEBUG(pf, 1,
6401 "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
6402 ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
6403 ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
6407 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6410 struct i40e_virtchnl_version_info reply;
6412 if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
6413 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
6418 vf->version = ((struct i40e_virtchnl_version_info *)msg)->minor;
6420 reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
6421 reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
6422 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
6427 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6431 if (msg_size != 0) {
6432 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
6437 ixl_reset_vf(pf, vf);
6439 /* No response to a reset message. */
6443 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6446 struct i40e_virtchnl_vf_resource reply;
6448 if ((vf->version == 0 && msg_size != 0) ||
6449 (vf->version == 1 && msg_size != 4)) {
6450 device_printf(pf->dev, "Invalid GET_VF_RESOURCES message size,"
6451 " for VF version %d.%d\n", I40E_VIRTCHNL_VERSION_MAJOR,
6453 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6458 bzero(&reply, sizeof(reply));
6460 if (vf->version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS)
6461 reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
6462 I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
6463 I40E_VIRTCHNL_VF_OFFLOAD_VLAN;
6465 reply.vf_offload_flags = *(u32 *)msg;
6468 reply.num_queue_pairs = vf->vsi.num_queues;
6469 reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
6470 reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
6471 reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
6472 reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
6473 memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
6475 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6476 I40E_SUCCESS, &reply, sizeof(reply));
6480 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6481 struct i40e_virtchnl_txq_info *info)
6484 struct i40e_hmc_obj_txq txq;
6485 uint16_t global_queue_num, global_vf_num;
6486 enum i40e_status_code status;
6490 global_queue_num = vf->vsi.first_queue + info->queue_id;
6491 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6492 bzero(&txq, sizeof(txq));
6494 status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
6495 if (status != I40E_SUCCESS)
6498 txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
6500 txq.head_wb_ena = info->headwb_enabled;
6501 txq.head_wb_addr = info->dma_headwb_addr;
6502 txq.qlen = info->ring_len;
6503 txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
6504 txq.rdylist_act = 0;
6506 status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
6507 if (status != I40E_SUCCESS)
6510 qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
6511 (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
6512 (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
6513 wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
6520 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6521 struct i40e_virtchnl_rxq_info *info)
6524 struct i40e_hmc_obj_rxq rxq;
6525 uint16_t global_queue_num;
6526 enum i40e_status_code status;
6529 global_queue_num = vf->vsi.first_queue + info->queue_id;
6530 bzero(&rxq, sizeof(rxq));
6532 if (info->databuffer_size > IXL_VF_MAX_BUFFER)
6535 if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
6536 info->max_pkt_size < ETHER_MIN_LEN)
6539 if (info->splithdr_enabled) {
6540 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
6543 rxq.hsplit_0 = info->rx_split_pos &
6544 (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
6545 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
6546 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
6547 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
6548 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
6553 status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
6554 if (status != I40E_SUCCESS)
6557 rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
6558 rxq.qlen = info->ring_len;
6560 rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
6566 rxq.rxmax = info->max_pkt_size;
6567 rxq.tphrdesc_ena = 1;
6568 rxq.tphwdesc_ena = 1;
6569 rxq.tphdata_ena = 1;
6570 rxq.tphhead_ena = 1;
6574 status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
6575 if (status != I40E_SUCCESS)
6582 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6585 struct i40e_virtchnl_vsi_queue_config_info *info;
6586 struct i40e_virtchnl_queue_pair_info *pair;
6589 if (msg_size < sizeof(*info)) {
6590 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6596 if (info->num_queue_pairs == 0) {
6597 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6602 if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
6603 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6608 if (info->vsi_id != vf->vsi.vsi_num) {
6609 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6614 for (i = 0; i < info->num_queue_pairs; i++) {
6615 pair = &info->qpair[i];
6617 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
6618 pair->rxq.vsi_id != vf->vsi.vsi_num ||
6619 pair->txq.queue_id != pair->rxq.queue_id ||
6620 pair->txq.queue_id >= vf->vsi.num_queues) {
6622 i40e_send_vf_nack(pf, vf,
6623 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6627 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
6628 i40e_send_vf_nack(pf, vf,
6629 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6633 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
6634 i40e_send_vf_nack(pf, vf,
6635 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6640 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
6644 ixl_vf_set_qctl(struct ixl_pf *pf,
6645 const struct i40e_virtchnl_vector_map *vector,
6646 enum i40e_queue_type cur_type, uint16_t cur_queue,
6647 enum i40e_queue_type *last_type, uint16_t *last_queue)
6649 uint32_t offset, qctl;
6652 if (cur_type == I40E_QUEUE_TYPE_RX) {
6653 offset = I40E_QINT_RQCTL(cur_queue);
6654 itr_indx = vector->rxitr_idx;
6656 offset = I40E_QINT_TQCTL(cur_queue);
6657 itr_indx = vector->txitr_idx;
6660 qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
6661 (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
6662 (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
6663 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
6664 (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
6666 wr32(&pf->hw, offset, qctl);
6668 *last_type = cur_type;
6669 *last_queue = cur_queue;
6673 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
6674 const struct i40e_virtchnl_vector_map *vector)
6678 enum i40e_queue_type type, last_type;
6679 uint32_t lnklst_reg;
6680 uint16_t rxq_map, txq_map, cur_queue, last_queue;
6684 rxq_map = vector->rxq_map;
6685 txq_map = vector->txq_map;
6687 last_queue = IXL_END_OF_INTR_LNKLST;
6688 last_type = I40E_QUEUE_TYPE_RX;
6691 * The datasheet says to optimize performance, RX queues and TX queues
6692 * should be interleaved in the interrupt linked list, so we process
6693 * both at once here.
6695 while ((rxq_map != 0) || (txq_map != 0)) {
6697 qindex = ffs(txq_map) - 1;
6698 type = I40E_QUEUE_TYPE_TX;
6699 cur_queue = vf->vsi.first_queue + qindex;
6700 ixl_vf_set_qctl(pf, vector, type, cur_queue,
6701 &last_type, &last_queue);
6702 txq_map &= ~(1 << qindex);
6706 qindex = ffs(rxq_map) - 1;
6707 type = I40E_QUEUE_TYPE_RX;
6708 cur_queue = vf->vsi.first_queue + qindex;
6709 ixl_vf_set_qctl(pf, vector, type, cur_queue,
6710 &last_type, &last_queue);
6711 rxq_map &= ~(1 << qindex);
6715 if (vector->vector_id == 0)
6716 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
6718 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
6720 wr32(hw, lnklst_reg,
6721 (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
6722 (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
6728 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6731 struct i40e_virtchnl_irq_map_info *map;
6732 struct i40e_virtchnl_vector_map *vector;
6734 int i, largest_txq, largest_rxq;
6738 if (msg_size < sizeof(*map)) {
6739 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6745 if (map->num_vectors == 0) {
6746 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6751 if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
6752 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6757 for (i = 0; i < map->num_vectors; i++) {
6758 vector = &map->vecmap[i];
6760 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
6761 vector->vsi_id != vf->vsi.vsi_num) {
6762 i40e_send_vf_nack(pf, vf,
6763 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
6767 if (vector->rxq_map != 0) {
6768 largest_rxq = fls(vector->rxq_map) - 1;
6769 if (largest_rxq >= vf->vsi.num_queues) {
6770 i40e_send_vf_nack(pf, vf,
6771 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6777 if (vector->txq_map != 0) {
6778 largest_txq = fls(vector->txq_map) - 1;
6779 if (largest_txq >= vf->vsi.num_queues) {
6780 i40e_send_vf_nack(pf, vf,
6781 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6787 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6788 vector->txitr_idx > IXL_MAX_ITR_IDX) {
6789 i40e_send_vf_nack(pf, vf,
6790 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6795 ixl_vf_config_vector(pf, vf, vector);
6798 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6802 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6805 struct i40e_virtchnl_queue_select *select;
6808 if (msg_size != sizeof(*select)) {
6809 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6815 if (select->vsi_id != vf->vsi.vsi_num ||
6816 select->rx_queues == 0 || select->tx_queues == 0) {
6817 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6822 error = ixl_enable_rings(&vf->vsi);
6824 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6829 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6833 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6834 void *msg, uint16_t msg_size)
6836 struct i40e_virtchnl_queue_select *select;
6839 if (msg_size != sizeof(*select)) {
6840 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6846 if (select->vsi_id != vf->vsi.vsi_num ||
6847 select->rx_queues == 0 || select->tx_queues == 0) {
6848 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6853 error = ixl_disable_rings(&vf->vsi);
6855 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6860 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6864 ixl_zero_mac(const uint8_t *addr)
6866 uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6868 return (cmp_etheraddr(addr, zero));
6872 ixl_bcast_mac(const uint8_t *addr)
6875 return (cmp_etheraddr(addr, ixl_bcast_addr));
6879 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6882 if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6886 * If the VF is not allowed to change its MAC address, don't let it
6887 * set a MAC filter for an address that is not a multicast address and
6888 * is not its assigned MAC.
6890 if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6891 !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6898 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6901 struct i40e_virtchnl_ether_addr_list *addr_list;
6902 struct i40e_virtchnl_ether_addr *addr;
6903 struct ixl_vsi *vsi;
6905 size_t expected_size;
6909 if (msg_size < sizeof(*addr_list)) {
6910 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6916 expected_size = sizeof(*addr_list) +
6917 addr_list->num_elements * sizeof(*addr);
6919 if (addr_list->num_elements == 0 ||
6920 addr_list->vsi_id != vsi->vsi_num ||
6921 msg_size != expected_size) {
6922 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6927 for (i = 0; i < addr_list->num_elements; i++) {
6928 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6929 i40e_send_vf_nack(pf, vf,
6930 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6935 for (i = 0; i < addr_list->num_elements; i++) {
6936 addr = &addr_list->list[i];
6937 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6940 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6944 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6947 struct i40e_virtchnl_ether_addr_list *addr_list;
6948 struct i40e_virtchnl_ether_addr *addr;
6949 size_t expected_size;
6952 if (msg_size < sizeof(*addr_list)) {
6953 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6959 expected_size = sizeof(*addr_list) +
6960 addr_list->num_elements * sizeof(*addr);
6962 if (addr_list->num_elements == 0 ||
6963 addr_list->vsi_id != vf->vsi.vsi_num ||
6964 msg_size != expected_size) {
6965 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6970 for (i = 0; i < addr_list->num_elements; i++) {
6971 addr = &addr_list->list[i];
6972 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6973 i40e_send_vf_nack(pf, vf,
6974 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6979 for (i = 0; i < addr_list->num_elements; i++) {
6980 addr = &addr_list->list[i];
6981 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6984 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6987 static enum i40e_status_code
6988 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6990 struct i40e_vsi_context vsi_ctx;
6992 vsi_ctx.seid = vf->vsi.seid;
6994 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6995 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6996 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6997 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6998 return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
7002 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7005 struct i40e_virtchnl_vlan_filter_list *filter_list;
7006 enum i40e_status_code code;
7007 size_t expected_size;
7010 if (msg_size < sizeof(*filter_list)) {
7011 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7017 expected_size = sizeof(*filter_list) +
7018 filter_list->num_elements * sizeof(uint16_t);
7019 if (filter_list->num_elements == 0 ||
7020 filter_list->vsi_id != vf->vsi.vsi_num ||
7021 msg_size != expected_size) {
7022 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7027 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7028 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7033 for (i = 0; i < filter_list->num_elements; i++) {
7034 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7035 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7041 code = ixl_vf_enable_vlan_strip(pf, vf);
7042 if (code != I40E_SUCCESS) {
7043 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7047 for (i = 0; i < filter_list->num_elements; i++)
7048 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7050 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
7054 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7057 struct i40e_virtchnl_vlan_filter_list *filter_list;
7059 size_t expected_size;
7061 if (msg_size < sizeof(*filter_list)) {
7062 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7068 expected_size = sizeof(*filter_list) +
7069 filter_list->num_elements * sizeof(uint16_t);
7070 if (filter_list->num_elements == 0 ||
7071 filter_list->vsi_id != vf->vsi.vsi_num ||
7072 msg_size != expected_size) {
7073 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7078 for (i = 0; i < filter_list->num_elements; i++) {
7079 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7080 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7086 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7087 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7092 for (i = 0; i < filter_list->num_elements; i++)
7093 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7095 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
7099 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
7100 void *msg, uint16_t msg_size)
7102 struct i40e_virtchnl_promisc_info *info;
7103 enum i40e_status_code code;
7105 if (msg_size != sizeof(*info)) {
7106 i40e_send_vf_nack(pf, vf,
7107 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7111 if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) {
7112 i40e_send_vf_nack(pf, vf,
7113 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7118 if (info->vsi_id != vf->vsi.vsi_num) {
7119 i40e_send_vf_nack(pf, vf,
7120 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7124 code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
7125 info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
7126 if (code != I40E_SUCCESS) {
7127 i40e_send_vf_nack(pf, vf,
7128 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7132 code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
7133 info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
7134 if (code != I40E_SUCCESS) {
7135 i40e_send_vf_nack(pf, vf,
7136 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7140 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
7144 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7147 struct i40e_virtchnl_queue_select *queue;
7149 if (msg_size != sizeof(*queue)) {
7150 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7156 if (queue->vsi_id != vf->vsi.vsi_num) {
7157 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7162 ixl_update_eth_stats(&vf->vsi);
7164 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7165 I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
7169 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
7173 uint16_t vf_num, msg_size;
7176 vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
7177 opcode = le32toh(event->desc.cookie_high);
7179 if (vf_num >= pf->num_vfs) {
7180 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
7184 vf = &pf->vfs[vf_num];
7185 msg = event->msg_buf;
7186 msg_size = event->msg_len;
7188 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
7189 "Got msg %s(%d) from VF-%d of size %d\n",
7190 ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
7193 case I40E_VIRTCHNL_OP_VERSION:
7194 ixl_vf_version_msg(pf, vf, msg, msg_size);
7196 case I40E_VIRTCHNL_OP_RESET_VF:
7197 ixl_vf_reset_msg(pf, vf, msg, msg_size);
7199 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
7200 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
7202 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
7203 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
7205 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
7206 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
7208 case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
7209 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
7211 case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
7212 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
7214 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
7215 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
7217 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
7218 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
7220 case I40E_VIRTCHNL_OP_ADD_VLAN:
7221 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
7223 case I40E_VIRTCHNL_OP_DEL_VLAN:
7224 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
7226 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
7227 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
7229 case I40E_VIRTCHNL_OP_GET_STATS:
7230 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
7233 /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
7234 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
7235 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
7237 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
7242 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
7244 ixl_handle_vflr(void *arg, int pending)
7248 uint16_t global_vf_num;
7249 uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
7256 for (i = 0; i < pf->num_vfs; i++) {
7257 global_vf_num = hw->func_caps.vf_base_id + i;
7259 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
7260 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
7261 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
7262 if (vflrstat & vflrstat_mask) {
7263 wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
7266 ixl_reinit_vf(pf, &pf->vfs[i]);
7270 icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
7271 icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
7272 wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
7279 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
7283 case I40E_AQ_RC_EPERM:
7285 case I40E_AQ_RC_ENOENT:
7287 case I40E_AQ_RC_ESRCH:
7289 case I40E_AQ_RC_EINTR:
7291 case I40E_AQ_RC_EIO:
7293 case I40E_AQ_RC_ENXIO:
7295 case I40E_AQ_RC_E2BIG:
7297 case I40E_AQ_RC_EAGAIN:
7299 case I40E_AQ_RC_ENOMEM:
7301 case I40E_AQ_RC_EACCES:
7303 case I40E_AQ_RC_EFAULT:
7305 case I40E_AQ_RC_EBUSY:
7307 case I40E_AQ_RC_EEXIST:
7309 case I40E_AQ_RC_EINVAL:
7311 case I40E_AQ_RC_ENOTTY:
7313 case I40E_AQ_RC_ENOSPC:
7315 case I40E_AQ_RC_ENOSYS:
7317 case I40E_AQ_RC_ERANGE:
7319 case I40E_AQ_RC_EFLUSHED:
7320 return (EINVAL); /* No exact equivalent in errno.h */
7321 case I40E_AQ_RC_BAD_ADDR:
7323 case I40E_AQ_RC_EMODE:
7325 case I40E_AQ_RC_EFBIG:
7333 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
7337 struct ixl_vsi *pf_vsi;
7338 enum i40e_status_code ret;
7341 pf = device_get_softc(dev);
7346 pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
7349 if (pf->vfs == NULL) {
7354 for (i = 0; i < num_vfs; i++)
7355 sysctl_ctx_init(&pf->vfs[i].ctx);
7357 ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
7358 1, FALSE, &pf->veb_seid, FALSE, NULL);
7359 if (ret != I40E_SUCCESS) {
7360 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
7361 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
7366 // TODO: [Configure MSI-X here]
7367 ixl_enable_adminq(hw);
7369 pf->num_vfs = num_vfs;
7374 free(pf->vfs, M_IXL);
7381 ixl_iov_uninit(device_t dev)
7385 struct ixl_vsi *vsi;
7390 pf = device_get_softc(dev);
7396 for (i = 0; i < pf->num_vfs; i++) {
7397 if (pf->vfs[i].vsi.seid != 0)
7398 i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
7401 if (pf->veb_seid != 0) {
7402 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
7406 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
7407 ixl_disable_intr(vsi);
7410 num_vfs = pf->num_vfs;
7416 /* Do this after the unlock as sysctl_ctx_free might sleep. */
7417 for (i = 0; i < num_vfs; i++)
7418 sysctl_ctx_free(&vfs[i].ctx);
7423 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
7425 char sysctl_name[QUEUE_NAME_LEN];
7432 pf = device_get_softc(dev);
7433 vf = &pf->vfs[vfnum];
7439 vf->vf_flags = VF_FLAG_ENABLED;
7440 SLIST_INIT(&vf->vsi.ftl);
7442 error = ixl_vf_setup_vsi(pf, vf);
7446 if (nvlist_exists_binary(params, "mac-addr")) {
7447 mac = nvlist_get_binary(params, "mac-addr", &size);
7448 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
7450 if (nvlist_get_bool(params, "allow-set-mac"))
7451 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7454 * If the administrator has not specified a MAC address then
7455 * we must allow the VF to choose one.
7457 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7459 if (nvlist_get_bool(params, "mac-anti-spoof"))
7460 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
7462 if (nvlist_get_bool(params, "allow-promisc"))
7463 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
7465 /* TODO: Get VLAN that PF has set for the VF */
7467 vf->vf_flags |= VF_FLAG_VLAN_CAP;
7469 ixl_reset_vf(pf, vf);
7473 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
7474 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
7479 #endif /* PCI_IOV */