sys/dev/ixl/if_ixl.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifndef IXL_STANDALONE_BUILD
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "opt_rss.h"
  39 #endif
  40
  41 #include "ixl.h"
  42 #include "ixl_pf.h"
  43
  44 #ifdef RSS
  45 #include <net/rss_config.h>
  46 #endif
  47
  48 /*********************************************************************
  49  *  Driver version
  50  *********************************************************************/
  51 char ixl_driver_version[] = "1.4.27-k";
  52
  53 /*********************************************************************
  54  *  PCI Device ID Table
  55  *
  56  *  Used by probe to select devices to load on
  57  *  Last field stores an index into ixl_strings
  58  *  Last entry must be all 0s
  59  *
  60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  61  *********************************************************************/
  62
  63 static ixl_vendor_info_t ixl_vendor_info_array[] =
  64 {
  65         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
  66         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
  67         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
  68         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
  69         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
  70         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
  71         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
  72         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
  73         /* required last entry */
  74         {0, 0, 0, 0, 0}
  75 };
  76
  77 /*********************************************************************
  78  *  Table of branding strings
  79  *********************************************************************/
  80
  81 static char    *ixl_strings[] = {
  82         "Intel(R) Ethernet Connection XL710 Driver"
  83 };
  84
  85
  86 /*********************************************************************
  87  *  Function prototypes
  88  *********************************************************************/
  89 static int      ixl_probe(device_t);
  90 static int      ixl_attach(device_t);
  91 static int      ixl_detach(device_t);
  92 static int      ixl_shutdown(device_t);
  93 static int      ixl_get_hw_capabilities(struct ixl_pf *);
  94 static void     ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
  95 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
  96 static void     ixl_init(void *);
  97 static void     ixl_init_locked(struct ixl_pf *);
  98 static void     ixl_stop(struct ixl_pf *);
  99 static void     ixl_stop_locked(struct ixl_pf *);
 100 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
 101 static int      ixl_media_change(struct ifnet *);
 102 static void     ixl_update_link_status(struct ixl_pf *);
 103 static int      ixl_allocate_pci_resources(struct ixl_pf *);
 104 static u16      ixl_get_bus_info(struct i40e_hw *, device_t);
 105 static int      ixl_setup_stations(struct ixl_pf *);
 106 static int      ixl_switch_config(struct ixl_pf *);
 107 static int      ixl_initialize_vsi(struct ixl_vsi *);
 108
 109 static int      ixl_setup_adminq_msix(struct ixl_pf *);
 110 static int      ixl_setup_adminq_tq(struct ixl_pf *);
 111 static int      ixl_setup_queue_msix(struct ixl_vsi *);
 112 static int      ixl_setup_queue_tqs(struct ixl_vsi *);
 113 static int      ixl_teardown_adminq_msix(struct ixl_pf *);
 114 static int      ixl_teardown_queue_msix(struct ixl_vsi *);
 115 static void     ixl_configure_intr0_msix(struct ixl_pf *);
 116 static void     ixl_configure_queue_intr_msix(struct ixl_pf *);
 117 static void     ixl_free_queue_tqs(struct ixl_vsi *);
 118 static void     ixl_free_adminq_tq(struct ixl_pf *);
 119
 120 static int      ixl_assign_vsi_legacy(struct ixl_pf *);
 121 static int      ixl_init_msix(struct ixl_pf *);
 122 static void     ixl_configure_itr(struct ixl_pf *);
 123 static void     ixl_configure_legacy(struct ixl_pf *);
 124 static void     ixl_free_pci_resources(struct ixl_pf *);
 125 static void     ixl_local_timer(void *);
 126 static int      ixl_setup_interface(device_t, struct ixl_vsi *);
 127 static void     ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
 128 static void     ixl_config_rss(struct ixl_vsi *);
 129 static void     ixl_set_queue_rx_itr(struct ixl_queue *);
 130 static void     ixl_set_queue_tx_itr(struct ixl_queue *);
 131 static int      ixl_set_advertised_speeds(struct ixl_pf *, int);
 132 static void     ixl_get_initial_advertised_speeds(struct ixl_pf *);
 133
 134 static int      ixl_enable_rings(struct ixl_vsi *);
 135 static int      ixl_disable_rings(struct ixl_vsi *);
 136 static void     ixl_enable_intr(struct ixl_vsi *);
 137 static void     ixl_disable_intr(struct ixl_vsi *);
 138 static void     ixl_disable_rings_intr(struct ixl_vsi *);
 139
 140 static void     ixl_enable_adminq(struct i40e_hw *);
 141 static void     ixl_disable_adminq(struct i40e_hw *);
 142 static void     ixl_enable_queue(struct i40e_hw *, int);
 143 static void     ixl_disable_queue(struct i40e_hw *, int);
 144 static void     ixl_enable_legacy(struct i40e_hw *);
 145 static void     ixl_disable_legacy(struct i40e_hw *);
 146
 147 static void     ixl_set_promisc(struct ixl_vsi *);
 148 static void     ixl_add_multi(struct ixl_vsi *);
 149 static void     ixl_del_multi(struct ixl_vsi *);
 150 static void     ixl_register_vlan(void *, struct ifnet *, u16);
 151 static void     ixl_unregister_vlan(void *, struct ifnet *, u16);
 152 static void     ixl_setup_vlan_filters(struct ixl_vsi *);
 153
 154 static void     ixl_init_filters(struct ixl_vsi *);
 155 static void     ixl_reconfigure_filters(struct ixl_vsi *vsi);
 156 static void     ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
 157 static void     ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
 158 static void     ixl_add_hw_filters(struct ixl_vsi *, int, int);
 159 static void     ixl_del_hw_filters(struct ixl_vsi *, int);
 160 static struct ixl_mac_filter *
 161                 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
 162 static void     ixl_add_mc_filter(struct ixl_vsi *, u8 *);
 163 static void     ixl_free_mac_filters(struct ixl_vsi *vsi);
 164
 165 /* Sysctls*/
 166 static void     ixl_add_device_sysctls(struct ixl_pf *);
 167
 168 static int      ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
 169 static int      ixl_set_advertise(SYSCTL_HANDLER_ARGS);
 170 static int      ixl_current_speed(SYSCTL_HANDLER_ARGS);
 171 static int      ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
 172
 173 #ifdef IXL_DEBUG_SYSCTL
 174 static int      ixl_debug_info(SYSCTL_HANDLER_ARGS);
 175 static void     ixl_print_debug_info(struct ixl_pf *);
 176
 177 static int      ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
 178 static int      ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
 179 static int      ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 180 static int      ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
 181 static int      ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
 182 #endif
 183
 184 /* The MSI/X Interrupt handlers */
 185 static void     ixl_intr(void *);
 186 static void     ixl_msix_que(void *);
 187 static void     ixl_msix_adminq(void *);
 188 static void     ixl_handle_mdd_event(struct ixl_pf *);
 189
 190 /* Deferred interrupt tasklets */
 191 static void     ixl_do_adminq(void *, int);
 192
 193 /* Statistics */
 194 static void     ixl_add_hw_stats(struct ixl_pf *);
 195 static void     ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
 196                     struct sysctl_oid_list *, struct i40e_hw_port_stats *);
 197 static void     ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
 198                     struct sysctl_oid_list *,
 199                     struct i40e_eth_stats *);
 200 static void     ixl_update_stats_counters(struct ixl_pf *);
 201 static void     ixl_update_eth_stats(struct ixl_vsi *);
 202 static void     ixl_update_vsi_stats(struct ixl_vsi *);
 203 static void     ixl_pf_reset_stats(struct ixl_pf *);
 204 static void     ixl_vsi_reset_stats(struct ixl_vsi *);
 205 static void     ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
 206                     u64 *, u64 *);
 207 static void     ixl_stat_update32(struct i40e_hw *, u32, bool,
 208                     u64 *, u64 *);
 209 /* NVM update */
 210 static int      ixl_handle_nvmupd_cmd(struct ixl_pf *, struct ifdrv *);
 211 static void     ixl_handle_empr_reset(struct ixl_pf *);
 212 static int      ixl_rebuild_hw_structs_after_reset(struct ixl_pf *);
 213
 214 /* Debug helper functions */
 215 #ifdef IXL_DEBUG
 216 static void     ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *);
 217 #endif
 218
 219 #ifdef PCI_IOV
 220 static int      ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
 221
 222 static int      ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
 223 static void     ixl_iov_uninit(device_t dev);
 224 static int      ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
 225
 226 static void     ixl_handle_vf_msg(struct ixl_pf *,
 227                     struct i40e_arq_event_info *);
 228 static void     ixl_handle_vflr(void *arg, int pending);
 229
 230 static void     ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 231 static void     ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 232 #endif
 233
 234 /*********************************************************************
 235  *  FreeBSD Device Interface Entry Points
 236  *********************************************************************/
 237
 238 static device_method_t ixl_methods[] = {
 239         /* Device interface */
 240         DEVMETHOD(device_probe, ixl_probe),
 241         DEVMETHOD(device_attach, ixl_attach),
 242         DEVMETHOD(device_detach, ixl_detach),
 243         DEVMETHOD(device_shutdown, ixl_shutdown),
 244 #ifdef PCI_IOV
 245         DEVMETHOD(pci_iov_init, ixl_iov_init),
 246         DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
 247         DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
 248 #endif
 249         {0, 0}
 250 };
 251
 252 static driver_t ixl_driver = {
 253         "ixl", ixl_methods, sizeof(struct ixl_pf),
 254 };
 255
 256 devclass_t ixl_devclass;
 257 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
 258
 259 MODULE_DEPEND(ixl, pci, 1, 1, 1);
 260 MODULE_DEPEND(ixl, ether, 1, 1, 1);
 261 #ifdef DEV_NETMAP
 262 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
 263 #endif /* DEV_NETMAP */
 264
 265 /*
 266 ** Global reset mutex
 267 */
 268 static struct mtx ixl_reset_mtx;
 269
 270 /*
 271 ** TUNEABLE PARAMETERS:
 272 */
 273
 274 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
 275                    "IXL driver parameters");
 276
 277 /*
 278  * MSIX should be the default for best performance,
 279  * but this allows it to be forced off for testing.
 280  */
 281 static int ixl_enable_msix = 1;
 282 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
 283 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
 284     "Enable MSI-X interrupts");
 285
 286 /*
 287 ** Number of descriptors per ring:
 288 **   - TX and RX are the same size
 289 */
 290 static int ixl_ringsz = DEFAULT_RING;
 291 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
 292 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
 293     &ixl_ringsz, 0, "Descriptor Ring Size");
 294
 295 /*
 296 ** This can be set manually, if left as 0 the
 297 ** number of queues will be calculated based
 298 ** on cpus and msix vectors available.
 299 */
 300 int ixl_max_queues = 0;
 301 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
 302 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
 303     &ixl_max_queues, 0, "Number of Queues");
 304
 305 /*
 306 ** Controls for Interrupt Throttling
 307 **      - true/false for dynamic adjustment
 308 **      - default values for static ITR
 309 */
 310 int ixl_dynamic_rx_itr = 1;
 311 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
 312 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
 313     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
 314
 315 int ixl_dynamic_tx_itr = 1;
 316 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
 317 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
 318     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
 319
 320 int ixl_rx_itr = IXL_ITR_8K;
 321 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
 322 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
 323     &ixl_rx_itr, 0, "RX Interrupt Rate");
 324
 325 int ixl_tx_itr = IXL_ITR_4K;
 326 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
 327 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
 328     &ixl_tx_itr, 0, "TX Interrupt Rate");
 329
 330 #ifdef IXL_FDIR
 331 static int ixl_enable_fdir = 1;
 332 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
 333 /* Rate at which we sample */
 334 int ixl_atr_rate = 20;
 335 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
 336 #endif
 337
 338 #ifdef DEV_NETMAP
 339 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
 340 #include <dev/netmap/if_ixl_netmap.h>
 341 #endif /* DEV_NETMAP */
 342
 343 static char *ixl_fc_string[6] = {
 344         "None",
 345         "Rx",
 346         "Tx",
 347         "Full",
 348         "Priority",
 349         "Default"
 350 };
 351
 352 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
 353
 354 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
 355     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 356
 357 /*********************************************************************
 358  *  Device identification routine
 359  *
 360  *  ixl_probe determines if the driver should be loaded on
 361  *  the hardware based on PCI vendor/device id of the device.
 362  *
 363  *  return BUS_PROBE_DEFAULT on success, positive on failure
 364  *********************************************************************/
 365
 366 static int
 367 ixl_probe(device_t dev)
 368 {
 369         ixl_vendor_info_t *ent;
 370
 371         u16     pci_vendor_id, pci_device_id;
 372         u16     pci_subvendor_id, pci_subdevice_id;
 373         char    device_name[256];
 374         static bool lock_init = FALSE;
 375
 376 #if 0
 377         INIT_DEBUGOUT("ixl_probe: begin");
 378 #endif
 379         pci_vendor_id = pci_get_vendor(dev);
 380         if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
 381                 return (ENXIO);
 382
 383         pci_device_id = pci_get_device(dev);
 384         pci_subvendor_id = pci_get_subvendor(dev);
 385         pci_subdevice_id = pci_get_subdevice(dev);
 386
 387         ent = ixl_vendor_info_array;
 388         while (ent->vendor_id != 0) {
 389                 if ((pci_vendor_id == ent->vendor_id) &&
 390                     (pci_device_id == ent->device_id) &&
 391
 392                     ((pci_subvendor_id == ent->subvendor_id) ||
 393                      (ent->subvendor_id == 0)) &&
 394
 395                     ((pci_subdevice_id == ent->subdevice_id) ||
 396                      (ent->subdevice_id == 0))) {
 397                         sprintf(device_name, "%s, Version - %s",
 398                                 ixl_strings[ent->index],
 399                                 ixl_driver_version);
 400                         device_set_desc_copy(dev, device_name);
 401                         /* One shot mutex init */
 402                         if (lock_init == FALSE) {
 403                                 lock_init = TRUE;
 404                                 mtx_init(&ixl_reset_mtx,
 405                                     "ixl_reset",
 406                                     "IXL RESET Lock", MTX_DEF);
 407                         }
 408                         return (BUS_PROBE_DEFAULT);
 409                 }
 410                 ent++;
 411         }
 412         return (ENXIO);
 413 }
 414
 415 /*********************************************************************
 416  *  Device initialization routine
 417  *
 418  *  The attach entry point is called when the driver is being loaded.
 419  *  This routine identifies the type of hardware, allocates all resources
 420  *  and initializes the hardware.
 421  *
 422  *  return 0 on success, positive on failure
 423  *********************************************************************/
 424
 425 static int
 426 ixl_attach(device_t dev)
 427 {
 428         struct ixl_pf   *pf;
 429         struct i40e_hw  *hw;
 430         struct ixl_vsi  *vsi;
 431         u16             bus;
 432         int             error = 0;
 433 #ifdef PCI_IOV
 434         nvlist_t        *pf_schema, *vf_schema;
 435         int             iov_error;
 436 #endif
 437
 438         INIT_DEBUGOUT("ixl_attach: begin");
 439
 440         /* Allocate, clear, and link in our primary soft structure */
 441         pf = device_get_softc(dev);
 442         pf->dev = pf->osdep.dev = dev;
 443         hw = &pf->hw;
 444
 445         /*
 446         ** Note this assumes we have a single embedded VSI,
 447         ** this could be enhanced later to allocate multiple
 448         */
 449         vsi = &pf->vsi;
 450         vsi->dev = pf->dev;
 451
 452         /* Core Lock Init*/
 453         IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
 454
 455         /* Set up the timer callout */
 456         callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
 457
 458         /* Save off the PCI information */
 459         hw->vendor_id = pci_get_vendor(dev);
 460         hw->device_id = pci_get_device(dev);
 461         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
 462         hw->subsystem_vendor_id =
 463             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 464         hw->subsystem_device_id =
 465             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 466
 467         hw->bus.device = pci_get_slot(dev);
 468         hw->bus.func = pci_get_function(dev);
 469
 470         pf->vc_debug_lvl = 1;
 471
 472         /* Do PCI setup - map BAR0, etc */
 473         if (ixl_allocate_pci_resources(pf)) {
 474                 device_printf(dev, "Allocation of PCI resources failed\n");
 475                 error = ENXIO;
 476                 goto err_out;
 477         }
 478
 479         /* Establish a clean starting point */
 480         i40e_clear_hw(hw);
 481         error = i40e_pf_reset(hw);
 482         if (error) {
 483                 device_printf(dev, "PF reset failure %d\n", error);
 484                 error = EIO;
 485                 goto err_out;
 486         }
 487
 488         /* Set admin queue parameters */
 489         hw->aq.num_arq_entries = IXL_AQ_LEN;
 490         hw->aq.num_asq_entries = IXL_AQ_LEN;
 491         hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
 492         hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
 493
 494         /* Initialize mac filter list for VSI */
 495         SLIST_INIT(&vsi->ftl);
 496
 497         /* Initialize the shared code */
 498         error = i40e_init_shared_code(hw);
 499         if (error) {
 500                 device_printf(dev, "Unable to initialize shared code, error %d\n",
 501                     error);
 502                 error = EIO;
 503                 goto err_out;
 504         }
 505
 506         /* Set up the admin queue */
 507         error = i40e_init_adminq(hw);
 508         if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
 509                 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
 510                     error);
 511                 error = EIO;
 512                 goto err_out;
 513         }
 514         ixl_print_nvm_version(pf);
 515
 516         if (error == I40E_ERR_FIRMWARE_API_VERSION) {
 517                 device_printf(dev, "The driver for the device stopped "
 518                     "because the NVM image is newer than expected.\n"
 519                     "You must install the most recent version of "
 520                     "the network driver.\n");
 521                 error = EIO;
 522                 goto err_out;
 523         }
 524
 525         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
 526             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
 527                 device_printf(dev, "The driver for the device detected "
 528                     "a newer version of the NVM image than expected.\n"
 529                     "Please install the most recent version of the network driver.\n");
 530         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
 531             hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
 532                 device_printf(dev, "The driver for the device detected "
 533                     "an older version of the NVM image than expected.\n"
 534                     "Please update the NVM image.\n");
 535
 536         /* Clear PXE mode */
 537         i40e_clear_pxe_mode(hw);
 538
 539         /* Get capabilities from the device */
 540         error = ixl_get_hw_capabilities(pf);
 541         if (error) {
 542                 device_printf(dev, "HW capabilities failure!\n");
 543                 goto err_get_cap;
 544         }
 545
 546         /* Set up host memory cache */
 547         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 548             hw->func_caps.num_rx_qp, 0, 0);
 549         if (error) {
 550                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
 551                 goto err_get_cap;
 552         }
 553
 554         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 555         if (error) {
 556                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
 557                 goto err_mac_hmc;
 558         }
 559
 560         /* Disable LLDP from the firmware for certain NVM versions */
 561         if (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
 562             (pf->hw.aq.fw_maj_ver < 4))
 563                 i40e_aq_stop_lldp(hw, TRUE, NULL);
 564
 565         i40e_get_mac_addr(hw, hw->mac.addr);
 566         error = i40e_validate_mac_addr(hw->mac.addr);
 567         if (error) {
 568                 device_printf(dev, "validate_mac_addr failed: %d\n", error);
 569                 goto err_mac_hmc;
 570         }
 571         bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
 572         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 573
 574         /* Set up VSI and queues */
 575         if (ixl_setup_stations(pf) != 0) {
 576                 device_printf(dev, "setup stations failed!\n");
 577                 error = ENOMEM;
 578                 goto err_mac_hmc;
 579         }
 580
 581         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 582             (hw->aq.fw_maj_ver < 4)) {
 583                 i40e_msec_delay(75);
 584                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 585                 if (error) {
 586                         device_printf(dev, "link restart failed, aq_err=%d\n",
 587                             pf->hw.aq.asq_last_status);
 588                         goto err_late;
 589                 }
 590         }
 591
 592         /* Determine link state */
 593         hw->phy.get_link_info = TRUE;
 594         i40e_get_link_status(hw, &pf->link_up);
 595
 596         /* Setup OS network interface / ifnet */
 597         if (ixl_setup_interface(dev, vsi) != 0) {
 598                 device_printf(dev, "interface setup failed!\n");
 599                 error = EIO;
 600                 goto err_late;
 601         }
 602
 603         error = ixl_switch_config(pf);
 604         if (error) {
 605                 device_printf(dev, "Initial ixl_switch_config() failed: %d\n",
 606                      error);
 607                 goto err_late;
 608         }
 609
 610         /* Limit PHY interrupts to link, autoneg, and modules failure */
 611         error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
 612             NULL);
 613         if (error) {
 614                 device_printf(dev, "i40e_aq_set_phy_mask() failed: err %d,"
 615                     " aq_err %d\n", error, hw->aq.asq_last_status);
 616                 goto err_late;
 617         }
 618
 619         /* Get the bus configuration and set the shared code's config */
 620         bus = ixl_get_bus_info(hw, dev);
 621         i40e_set_pci_config_data(hw, bus);
 622
 623         /*
 624          * In MSI-X mode, initialize the Admin Queue interrupt,
 625          * so userland tools can communicate with the adapter regardless of
 626          * the ifnet interface's status.
 627          */
 628         if (pf->msix > 1) {
 629                 error = ixl_setup_adminq_msix(pf);
 630                 if (error) {
 631                         device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
 632                             error);
 633                         goto err_late;
 634                 }
 635                 error = ixl_setup_adminq_tq(pf);
 636                 if (error) {
 637                         device_printf(dev, "ixl_setup_adminq_tq error: %d\n",
 638                             error);
 639                         goto err_late;
 640                 }
 641                 ixl_configure_intr0_msix(pf);
 642                 ixl_enable_adminq(hw);
 643         }
 644
 645         /* Initialize statistics & add sysctls */
 646         ixl_add_device_sysctls(pf);
 647
 648         ixl_pf_reset_stats(pf);
 649         ixl_update_stats_counters(pf);
 650         ixl_add_hw_stats(pf);
 651
 652         /* Register for VLAN events */
 653         vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 654             ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 655         vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 656             ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 657
 658 #ifdef PCI_IOV
 659         /* SR-IOV is only supported when MSI-X is in use. */
 660         if (pf->msix > 1) {
 661                 pf_schema = pci_iov_schema_alloc_node();
 662                 vf_schema = pci_iov_schema_alloc_node();
 663                 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
 664                 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
 665                     IOV_SCHEMA_HASDEFAULT, TRUE);
 666                 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
 667                     IOV_SCHEMA_HASDEFAULT, FALSE);
 668                 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
 669                     IOV_SCHEMA_HASDEFAULT, FALSE);
 670
 671                 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
 672                 if (iov_error != 0) {
 673                         device_printf(dev,
 674                             "Failed to initialize SR-IOV (error=%d)\n",
 675                             iov_error);
 676                 } else
 677                         device_printf(dev, "SR-IOV ready\n");
 678         }
 679 #endif
 680
 681 #ifdef DEV_NETMAP
 682         ixl_netmap_attach(vsi);
 683 #endif /* DEV_NETMAP */
 684         INIT_DEBUGOUT("ixl_attach: end");
 685         return (0);
 686
 687 err_late:
 688         if (vsi->ifp != NULL)
 689                 if_free(vsi->ifp);
 690 err_mac_hmc:
 691         i40e_shutdown_lan_hmc(hw);
 692 err_get_cap:
 693         i40e_shutdown_adminq(hw);
 694 err_out:
 695         ixl_free_pci_resources(pf);
 696         ixl_free_vsi(vsi);
 697         IXL_PF_LOCK_DESTROY(pf);
 698         return (error);
 699 }
 700
 701 /*********************************************************************
 702  *  Device removal routine
 703  *
 704  *  The detach entry point is called when the driver is being removed.
 705  *  This routine stops the adapter and deallocates all the resources
 706  *  that were allocated for driver operation.
 707  *
 708  *  return 0 on success, positive on failure
 709  *********************************************************************/
 710
 711 static int
 712 ixl_detach(device_t dev)
 713 {
 714         struct ixl_pf           *pf = device_get_softc(dev);
 715         struct i40e_hw          *hw = &pf->hw;
 716         struct ixl_vsi          *vsi = &pf->vsi;
 717         enum i40e_status_code   status;
 718 #ifdef PCI_IOV
 719         int                     error;
 720 #endif
 721
 722         INIT_DEBUGOUT("ixl_detach: begin");
 723
 724         /* Make sure VLANS are not using driver */
 725         if (vsi->ifp->if_vlantrunk != NULL) {
 726                 device_printf(dev, "Vlan in use, detach first\n");
 727                 return (EBUSY);
 728         }
 729
 730 #ifdef PCI_IOV
 731         error = pci_iov_detach(dev);
 732         if (error != 0) {
 733                 device_printf(dev, "SR-IOV in use; detach first.\n");
 734                 return (error);
 735         }
 736 #endif
 737
 738         ether_ifdetach(vsi->ifp);
 739         if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)
 740                 ixl_stop(pf);
 741
 742         ixl_free_queue_tqs(vsi);
 743
 744         /* Shutdown LAN HMC */
 745         status = i40e_shutdown_lan_hmc(hw);
 746         if (status)
 747                 device_printf(dev,
 748                     "Shutdown LAN HMC failed with code %d\n", status);
 749
 750         /* Shutdown admin queue */
 751         ixl_disable_adminq(hw);
 752         ixl_free_adminq_tq(pf);
 753         ixl_teardown_adminq_msix(pf);
 754         status = i40e_shutdown_adminq(hw);
 755         if (status)
 756                 device_printf(dev,
 757                     "Shutdown Admin queue failed with code %d\n", status);
 758
 759         /* Unregister VLAN events */
 760         if (vsi->vlan_attach != NULL)
 761                 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
 762         if (vsi->vlan_detach != NULL)
 763                 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
 764
 765         callout_drain(&pf->timer);
 766 #ifdef DEV_NETMAP
 767         netmap_detach(vsi->ifp);
 768 #endif /* DEV_NETMAP */
 769         ixl_free_pci_resources(pf);
 770         bus_generic_detach(dev);
 771         if_free(vsi->ifp);
 772         ixl_free_vsi(vsi);
 773         IXL_PF_LOCK_DESTROY(pf);
 774         return (0);
 775 }
 776
 777 /*********************************************************************
 778  *
 779  *  Shutdown entry point
 780  *
 781  **********************************************************************/
 782
 783 static int
 784 ixl_shutdown(device_t dev)
 785 {
 786         struct ixl_pf *pf = device_get_softc(dev);
 787         ixl_stop(pf);
 788         return (0);
 789 }
 790
 791
 792 /*********************************************************************
 793  *
 794  *  Get the hardware capabilities
 795  *
 796  **********************************************************************/
 797
 798 static int
 799 ixl_get_hw_capabilities(struct ixl_pf *pf)
 800 {
 801         struct i40e_aqc_list_capabilities_element_resp *buf;
 802         struct i40e_hw  *hw = &pf->hw;
 803         device_t        dev = pf->dev;
 804         int             error, len;
 805         u16             needed;
 806         bool            again = TRUE;
 807
 808         len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
 809 retry:
 810         if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
 811             malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 812                 device_printf(dev, "Unable to allocate cap memory\n");
 813                 return (ENOMEM);
 814         }
 815
 816         /* This populates the hw struct */
 817         error = i40e_aq_discover_capabilities(hw, buf, len,
 818             &needed, i40e_aqc_opc_list_func_capabilities, NULL);
 819         free(buf, M_DEVBUF);
 820         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
 821             (again == TRUE)) {
 822                 /* retry once with a larger buffer */
 823                 again = FALSE;
 824                 len = needed;
 825                 goto retry;
 826         } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
 827                 device_printf(dev, "capability discovery failed: %d\n",
 828                     pf->hw.aq.asq_last_status);
 829                 return (ENODEV);
 830         }
 831
 832         /* Capture this PF's starting queue pair */
 833         pf->qbase = hw->func_caps.base_queue;
 834
 835 #ifdef IXL_DEBUG
 836         device_printf(dev, "pf_id=%d, num_vfs=%d, msix_pf=%d, "
 837             "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
 838             hw->pf_id, hw->func_caps.num_vfs,
 839             hw->func_caps.num_msix_vectors,
 840             hw->func_caps.num_msix_vectors_vf,
 841             hw->func_caps.fd_filters_guaranteed,
 842             hw->func_caps.fd_filters_best_effort,
 843             hw->func_caps.num_tx_qp,
 844             hw->func_caps.num_rx_qp,
 845             hw->func_caps.base_queue);
 846 #endif
 847         return (error);
 848 }
 849
 850 static void
 851 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
 852 {
 853         device_t        dev = vsi->dev;
 854
 855         /* Enable/disable TXCSUM/TSO4 */
 856         if (!(ifp->if_capenable & IFCAP_TXCSUM)
 857             && !(ifp->if_capenable & IFCAP_TSO4)) {
 858                 if (mask & IFCAP_TXCSUM) {
 859                         ifp->if_capenable |= IFCAP_TXCSUM;
 860                         /* enable TXCSUM, restore TSO if previously enabled */
 861                         if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
 862                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 863                                 ifp->if_capenable |= IFCAP_TSO4;
 864                         }
 865                 }
 866                 else if (mask & IFCAP_TSO4) {
 867                         ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
 868                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 869                         device_printf(dev,
 870                             "TSO4 requires txcsum, enabling both...\n");
 871                 }
 872         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 873             && !(ifp->if_capenable & IFCAP_TSO4)) {
 874                 if (mask & IFCAP_TXCSUM)
 875                         ifp->if_capenable &= ~IFCAP_TXCSUM;
 876                 else if (mask & IFCAP_TSO4)
 877                         ifp->if_capenable |= IFCAP_TSO4;
 878         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 879             && (ifp->if_capenable & IFCAP_TSO4)) {
 880                 if (mask & IFCAP_TXCSUM) {
 881                         vsi->flags |= IXL_FLAGS_KEEP_TSO4;
 882                         ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
 883                         device_printf(dev,
 884                             "TSO4 requires txcsum, disabling both...\n");
 885                 } else if (mask & IFCAP_TSO4)
 886                         ifp->if_capenable &= ~IFCAP_TSO4;
 887         }
 888
 889         /* Enable/disable TXCSUM_IPV6/TSO6 */
 890         if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 891             && !(ifp->if_capenable & IFCAP_TSO6)) {
 892                 if (mask & IFCAP_TXCSUM_IPV6) {
 893                         ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
 894                         if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
 895                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 896                                 ifp->if_capenable |= IFCAP_TSO6;
 897                         }
 898                 } else if (mask & IFCAP_TSO6) {
 899                         ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 900                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 901                         device_printf(dev,
 902                             "TSO6 requires txcsum6, enabling both...\n");
 903                 }
 904         } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 905             && !(ifp->if_capenable & IFCAP_TSO6)) {
 906                 if (mask & IFCAP_TXCSUM_IPV6)
 907                         ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
 908                 else if (mask & IFCAP_TSO6)
 909                         ifp->if_capenable |= IFCAP_TSO6;
 910         } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 911             && (ifp->if_capenable & IFCAP_TSO6)) {
 912                 if (mask & IFCAP_TXCSUM_IPV6) {
 913                         vsi->flags |= IXL_FLAGS_KEEP_TSO6;
 914                         ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 915                         device_printf(dev,
 916                             "TSO6 requires txcsum6, disabling both...\n");
 917                 } else if (mask & IFCAP_TSO6)
 918                         ifp->if_capenable &= ~IFCAP_TSO6;
 919         }
 920 }
 921
 922 /*********************************************************************
 923  *  Ioctl entry point
 924  *
 925  *  ixl_ioctl is called when the user wants to configure the
 926  *  interface.
 927  *
 928  *  return 0 on success, positive on failure
 929  **********************************************************************/
 930
 931 static int
 932 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 933 {
 934         struct ixl_vsi  *vsi = ifp->if_softc;
 935         struct ixl_pf   *pf = vsi->back;
 936         struct ifreq    *ifr = (struct ifreq *)data;
 937         struct ifdrv    *ifd = (struct ifdrv *)data;
 938 #if defined(INET) || defined(INET6)
 939         struct ifaddr *ifa = (struct ifaddr *)data;
 940         bool            avoid_reset = FALSE;
 941 #endif
 942         int             error = 0;
 943
 944         switch (command) {
 945
 946         case SIOCSIFADDR:
 947 #ifdef INET
 948                 if (ifa->ifa_addr->sa_family == AF_INET)
 949                         avoid_reset = TRUE;
 950 #endif
 951 #ifdef INET6
 952                 if (ifa->ifa_addr->sa_family == AF_INET6)
 953                         avoid_reset = TRUE;
 954 #endif
 955 #if defined(INET) || defined(INET6)
 956                 /*
 957                 ** Calling init results in link renegotiation,
 958                 ** so we avoid doing it when possible.
 959                 */
 960                 if (avoid_reset) {
 961                         ifp->if_flags |= IFF_UP;
 962                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 963                                 ixl_init(pf);
 964 #ifdef INET
 965                         if (!(ifp->if_flags & IFF_NOARP))
 966                                 arp_ifinit(ifp, ifa);
 967 #endif
 968                 } else
 969                         error = ether_ioctl(ifp, command, data);
 970                 break;
 971 #endif
 972         case SIOCSIFMTU:
 973                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 974                 if (ifr->ifr_mtu > IXL_MAX_FRAME -
 975                    ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
 976                         error = EINVAL;
 977                 } else {
 978                         IXL_PF_LOCK(pf);
 979                         ifp->if_mtu = ifr->ifr_mtu;
 980                         vsi->max_frame_size =
 981                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
 982                             + ETHER_VLAN_ENCAP_LEN;
 983                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 984                                 ixl_init_locked(pf);
 985                         IXL_PF_UNLOCK(pf);
 986                 }
 987                 break;
 988         case SIOCSIFFLAGS:
 989                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 990                 IXL_PF_LOCK(pf);
 991                 if (ifp->if_flags & IFF_UP) {
 992                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 993                                 if ((ifp->if_flags ^ pf->if_flags) &
 994                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 995                                         ixl_set_promisc(vsi);
 996                                 }
 997                         } else {
 998                                 IXL_PF_UNLOCK(pf);
 999                                 ixl_init(pf);
1000                                 IXL_PF_LOCK(pf);
1001                         }
1002                 } else {
1003                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1004                                 IXL_PF_UNLOCK(pf);
1005                                 ixl_stop(pf);
1006                                 IXL_PF_LOCK(pf);
1007                         }
1008                 }
1009                 pf->if_flags = ifp->if_flags;
1010                 IXL_PF_UNLOCK(pf);
1011                 break;
1012         case SIOCSDRVSPEC:
1013         case SIOCGDRVSPEC:
1014                 IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific "
1015                     "Info)\n");
1016
1017                 /* NVM update command */
1018                 if (ifd->ifd_cmd == I40E_NVM_ACCESS)
1019                         error = ixl_handle_nvmupd_cmd(pf, ifd);
1020                 else
1021                         error = EINVAL;
1022                 break;
1023         case SIOCADDMULTI:
1024                 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1025                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1026                         IXL_PF_LOCK(pf);
1027                         ixl_disable_intr(vsi);
1028                         ixl_add_multi(vsi);
1029                         ixl_enable_intr(vsi);
1030                         IXL_PF_UNLOCK(pf);
1031                 }
1032                 break;
1033         case SIOCDELMULTI:
1034                 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1035                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1036                         IXL_PF_LOCK(pf);
1037                         ixl_disable_intr(vsi);
1038                         ixl_del_multi(vsi);
1039                         ixl_enable_intr(vsi);
1040                         IXL_PF_UNLOCK(pf);
1041                 }
1042                 break;
1043         case SIOCSIFMEDIA:
1044         case SIOCGIFMEDIA:
1045 #ifdef IFM_ETH_XTYPE
1046         case SIOCGIFXMEDIA:
1047 #endif
1048                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1049                 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1050                 break;
1051         case SIOCSIFCAP:
1052         {
1053                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1054                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1055
1056                 ixl_cap_txcsum_tso(vsi, ifp, mask);
1057
1058                 if (mask & IFCAP_RXCSUM)
1059                         ifp->if_capenable ^= IFCAP_RXCSUM;
1060                 if (mask & IFCAP_RXCSUM_IPV6)
1061                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1062                 if (mask & IFCAP_LRO)
1063                         ifp->if_capenable ^= IFCAP_LRO;
1064                 if (mask & IFCAP_VLAN_HWTAGGING)
1065                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1066                 if (mask & IFCAP_VLAN_HWFILTER)
1067                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1068                 if (mask & IFCAP_VLAN_HWTSO)
1069                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1070                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1071                         IXL_PF_LOCK(pf);
1072                         ixl_init_locked(pf);
1073                         IXL_PF_UNLOCK(pf);
1074                 }
1075                 VLAN_CAPABILITIES(ifp);
1076
1077                 break;
1078         }
1079
1080         default:
1081                 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1082                 error = ether_ioctl(ifp, command, data);
1083                 break;
1084         }
1085
1086         return (error);
1087 }
1088
1089
1090 /*********************************************************************
1091  *  Init entry point
1092  *
1093  *  This routine is used in two ways. It is used by the stack as
1094  *  init entry point in network interface structure. It is also used
1095  *  by the driver as a hw/sw initialization routine to get to a
1096  *  consistent state.
1097  *
1098  *  return 0 on success, positive on failure
1099  **********************************************************************/
1100
1101 static void
1102 ixl_init_locked(struct ixl_pf *pf)
1103 {
1104         struct i40e_hw  *hw = &pf->hw;
1105         struct ixl_vsi  *vsi = &pf->vsi;
1106         struct ifnet    *ifp = vsi->ifp;
1107         device_t        dev = pf->dev;
1108         struct i40e_filter_control_settings     filter;
1109         u8              tmpaddr[ETHER_ADDR_LEN];
1110         int             ret;
1111
1112         mtx_assert(&pf->pf_mtx, MA_OWNED);
1113         INIT_DEBUGOUT("ixl_init_locked: begin");
1114
1115         ixl_stop_locked(pf);
1116
1117         /* Get the latest mac address... User might use a LAA */
1118         bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1119               I40E_ETH_LENGTH_OF_ADDRESS);
1120         if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1121             (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
1122                 ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1123                 bcopy(tmpaddr, hw->mac.addr,
1124                     I40E_ETH_LENGTH_OF_ADDRESS);
1125                 ret = i40e_aq_mac_address_write(hw,
1126                     I40E_AQC_WRITE_TYPE_LAA_ONLY,
1127                     hw->mac.addr, NULL);
1128                 if (ret) {
1129                         device_printf(dev, "LLA address"
1130                          "change failed!!\n");
1131                         return;
1132                 }
1133         }
1134
1135         ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1136
1137         /* Set the various hardware offload abilities */
1138         ifp->if_hwassist = 0;
1139         if (ifp->if_capenable & IFCAP_TSO)
1140                 ifp->if_hwassist |= CSUM_TSO;
1141         if (ifp->if_capenable & IFCAP_TXCSUM)
1142                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1143         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1144                 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1145
1146         /* Set up the device filtering */
1147         bzero(&filter, sizeof(filter));
1148         filter.enable_ethtype = TRUE;
1149         filter.enable_macvlan = TRUE;
1150 #ifdef IXL_FDIR
1151         filter.enable_fdir = TRUE;
1152 #endif
1153         filter.hash_lut_size = I40E_HASH_LUT_SIZE_512;
1154         if (i40e_set_filter_control(hw, &filter))
1155                 device_printf(dev, "i40e_set_filter_control() failed\n");
1156
1157         /* Set up RSS */
1158         ixl_config_rss(vsi);
1159
1160         /* Prepare the VSI: rings, hmc contexts, etc... */
1161         if (ixl_initialize_vsi(vsi)) {
1162                 device_printf(dev, "initialize vsi failed!!\n");
1163                 return;
1164         }
1165
1166         /* Add protocol filters to list */
1167         ixl_init_filters(vsi);
1168
1169         /* Setup vlan's if needed */
1170         ixl_setup_vlan_filters(vsi);
1171
1172         /* Set up MSI/X routing and the ITR settings */
1173         if (ixl_enable_msix) {
1174                 ixl_configure_queue_intr_msix(pf);
1175                 ixl_configure_itr(pf);
1176         } else
1177                 ixl_configure_legacy(pf);
1178
1179         ixl_enable_rings(vsi);
1180
1181         i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1182
1183         ixl_reconfigure_filters(vsi);
1184
1185         /* And now turn on interrupts */
1186         ixl_enable_intr(vsi);
1187
1188         /* Get link info */
1189         hw->phy.get_link_info = TRUE;
1190         i40e_get_link_status(hw, &pf->link_up);
1191         ixl_update_link_status(pf);
1192
1193         /* Set initial advertised speed sysctl value */
1194         ixl_get_initial_advertised_speeds(pf);
1195
1196         /* Start the local timer */
1197         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1198
1199         /* Now inform the stack we're ready */
1200         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1201
1202         return;
1203 }
1204
1205 /* For the set_advertise sysctl */
1206 static void
1207 ixl_get_initial_advertised_speeds(struct ixl_pf *pf)
1208 {
1209         struct i40e_hw *hw = &pf->hw;
1210         device_t dev = pf->dev;
1211         enum i40e_status_code status;
1212         struct i40e_aq_get_phy_abilities_resp abilities;
1213
1214         /* Set initial sysctl values */
1215         status = i40e_aq_get_phy_capabilities(hw, FALSE, false, &abilities,
1216                                               NULL);
1217         if (status) {
1218                 /* Non-fatal error */
1219                 device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n",
1220                      __func__, status);
1221                 return;
1222         }
1223
1224         if (abilities.link_speed & I40E_LINK_SPEED_40GB)
1225                 pf->advertised_speed |= 0x10;
1226         if (abilities.link_speed & I40E_LINK_SPEED_20GB)
1227                 pf->advertised_speed |= 0x8;
1228         if (abilities.link_speed & I40E_LINK_SPEED_10GB)
1229                 pf->advertised_speed |= 0x4;
1230         if (abilities.link_speed & I40E_LINK_SPEED_1GB)
1231                 pf->advertised_speed |= 0x2;
1232         if (abilities.link_speed & I40E_LINK_SPEED_100MB)
1233                 pf->advertised_speed |= 0x1;
1234 }
1235
1236 static int
1237 ixl_teardown_hw_structs(struct ixl_pf *pf)
1238 {
1239         enum i40e_status_code status = 0;
1240         struct i40e_hw *hw = &pf->hw;
1241         device_t dev = pf->dev;
1242
1243         /* Shutdown LAN HMC */
1244         if (hw->hmc.hmc_obj) {
1245                 status = i40e_shutdown_lan_hmc(hw);
1246                 if (status) {
1247                         device_printf(dev,
1248                             "init: LAN HMC shutdown failure; status %d\n", status);
1249                         goto err_out;
1250                 }
1251         }
1252
1253         // XXX: This gets called when we know the adminq is inactive;
1254         // so we already know it's setup when we get here.
1255
1256         /* Shutdown admin queue */
1257         status = i40e_shutdown_adminq(hw);
1258         if (status)
1259                 device_printf(dev,
1260                     "init: Admin Queue shutdown failure; status %d\n", status);
1261
1262 err_out:
1263         return (status);
1264 }
1265
1266 static int
1267 ixl_reset(struct ixl_pf *pf)
1268 {
1269         struct i40e_hw *hw = &pf->hw;
1270         device_t dev = pf->dev;
1271         u8 set_fc_err_mask;
1272         int error = 0;
1273
1274         // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary
1275         i40e_clear_hw(hw);
1276         error = i40e_pf_reset(hw);
1277         if (error) {
1278                 device_printf(dev, "init: PF reset failure");
1279                 error = EIO;
1280                 goto err_out;
1281         }
1282
1283         error = i40e_init_adminq(hw);
1284         if (error) {
1285                 device_printf(dev, "init: Admin queue init failure;"
1286                     " status code %d", error);
1287                 error = EIO;
1288                 goto err_out;
1289         }
1290
1291         i40e_clear_pxe_mode(hw);
1292
1293         error = ixl_get_hw_capabilities(pf);
1294         if (error) {
1295                 device_printf(dev, "init: Error retrieving HW capabilities;"
1296                     " status code %d\n", error);
1297                 goto err_out;
1298         }
1299
1300         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1301             hw->func_caps.num_rx_qp, 0, 0);
1302         if (error) {
1303                 device_printf(dev, "init: LAN HMC init failed; status code %d\n",
1304                     error);
1305                 error = EIO;
1306                 goto err_out;
1307         }
1308
1309         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1310         if (error) {
1311                 device_printf(dev, "init: LAN HMC config failed; status code %d\n",
1312                     error);
1313                 error = EIO;
1314                 goto err_out;
1315         }
1316
1317         // XXX: possible fix for panic, but our failure recovery is still broken
1318         error = ixl_switch_config(pf);
1319         if (error) {
1320                 device_printf(dev, "init: ixl_switch_config() failed: %d\n",
1321                      error);
1322                 goto err_out;
1323         }
1324
1325         error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
1326             NULL);
1327         if (error) {
1328                 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
1329                     " aq_err %d\n", error, hw->aq.asq_last_status);
1330                 error = EIO;
1331                 goto err_out;
1332         }
1333
1334         error = i40e_set_fc(hw, &set_fc_err_mask, true);
1335         if (error) {
1336                 device_printf(dev, "init: setting link flow control failed; retcode %d,"
1337                     " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
1338                 goto err_out;
1339         }
1340
1341         // XXX: (Rebuild VSIs?)
1342
1343         /* Firmware delay workaround */
1344         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
1345             (hw->aq.fw_maj_ver < 4)) {
1346                 i40e_msec_delay(75);
1347                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
1348                 if (error) {
1349                         device_printf(dev, "init: link restart failed, aq_err %d\n",
1350                             hw->aq.asq_last_status);
1351                         goto err_out;
1352                 }
1353         }
1354
1355
1356 err_out:
1357         return (error);
1358 }
1359
1360 static void
1361 ixl_init(void *arg)
1362 {
1363         struct ixl_pf *pf = arg;
1364         struct ixl_vsi *vsi = &pf->vsi;
1365         device_t dev = pf->dev;
1366         int error = 0;
1367
1368         /*
1369          * If the aq is dead here, it probably means something outside of the driver
1370          * did something to the adapter, like a PF reset.
1371          * So rebuild the driver's state here if that occurs.
1372          */
1373         if (!i40e_check_asq_alive(&pf->hw)) {
1374                 device_printf(dev, "Admin Queue is down; resetting...\n");
1375                 IXL_PF_LOCK(pf);
1376                 ixl_teardown_hw_structs(pf);
1377                 ixl_reset(pf);
1378                 IXL_PF_UNLOCK(pf);
1379         }
1380
1381         /*
1382          * Set up LAN queue interrupts here.
1383          * Kernel interrupt setup functions cannot be called while holding a lock,
1384          * so this is done outside of init_locked().
1385          */
1386         if (pf->msix > 1) {
1387                 /* Teardown existing interrupts, if they exist */
1388                 ixl_teardown_queue_msix(vsi);
1389                 ixl_free_queue_tqs(vsi);
1390                 /* Then set them up again */
1391                 error = ixl_setup_queue_msix(vsi);
1392                 if (error)
1393                         device_printf(dev, "ixl_setup_queue_msix() error: %d\n",
1394                             error);
1395                 error = ixl_setup_queue_tqs(vsi);
1396                 if (error)
1397                         device_printf(dev, "ixl_setup_queue_tqs() error: %d\n",
1398                             error);
1399         } else
1400                 // possibly broken
1401                 error = ixl_assign_vsi_legacy(pf);
1402         if (error) {
1403                 device_printf(pf->dev, "assign_vsi_msix/legacy error: %d\n", error);
1404                 return;
1405         }
1406
1407         IXL_PF_LOCK(pf);
1408         ixl_init_locked(pf);
1409         IXL_PF_UNLOCK(pf);
1410 }
1411
1412 /*
1413 ** MSIX Interrupt Handlers and Tasklets
1414 */
1415 static void
1416 ixl_handle_que(void *context, int pending)
1417 {
1418         struct ixl_queue *que = context;
1419         struct ixl_vsi *vsi = que->vsi;
1420         struct i40e_hw  *hw = vsi->hw;
1421         struct tx_ring  *txr = &que->txr;
1422         struct ifnet    *ifp = vsi->ifp;
1423         bool            more;
1424
1425         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1426                 more = ixl_rxeof(que, IXL_RX_LIMIT);
1427                 IXL_TX_LOCK(txr);
1428                 ixl_txeof(que);
1429                 if (!drbr_empty(ifp, txr->br))
1430                         ixl_mq_start_locked(ifp, txr);
1431                 IXL_TX_UNLOCK(txr);
1432                 if (more) {
1433                         taskqueue_enqueue(que->tq, &que->task);
1434                         return;
1435                 }
1436         }
1437
1438         /* Reenable this interrupt - hmmm */
1439         ixl_enable_queue(hw, que->me);
1440         return;
1441 }
1442
1443
1444 /*********************************************************************
1445  *
1446  *  Legacy Interrupt Service routine
1447  *
1448  **********************************************************************/
1449 void
1450 ixl_intr(void *arg)
1451 {
1452         struct ixl_pf           *pf = arg;
1453         struct i40e_hw          *hw =  &pf->hw;
1454         struct ixl_vsi          *vsi = &pf->vsi;
1455         struct ixl_queue        *que = vsi->queues;
1456         struct ifnet            *ifp = vsi->ifp;
1457         struct tx_ring          *txr = &que->txr;
1458         u32                     reg, icr0, mask;
1459         bool                    more_tx, more_rx;
1460
1461         ++que->irqs;
1462
1463         /* Protect against spurious interrupts */
1464         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1465                 return;
1466
1467         icr0 = rd32(hw, I40E_PFINT_ICR0);
1468
1469         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1470         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1471         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1472
1473         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1474
1475 #ifdef PCI_IOV
1476         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1477                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1478 #endif
1479
1480         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1481                 taskqueue_enqueue(pf->tq, &pf->adminq);
1482                 return;
1483         }
1484
1485         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1486
1487         IXL_TX_LOCK(txr);
1488         more_tx = ixl_txeof(que);
1489         if (!drbr_empty(vsi->ifp, txr->br))
1490                 more_tx = 1;
1491         IXL_TX_UNLOCK(txr);
1492
1493         /* re-enable other interrupt causes */
1494         wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1495
1496         /* And now the queues */
1497         reg = rd32(hw, I40E_QINT_RQCTL(0));
1498         reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1499         wr32(hw, I40E_QINT_RQCTL(0), reg);
1500
1501         reg = rd32(hw, I40E_QINT_TQCTL(0));
1502         reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1503         reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1504         wr32(hw, I40E_QINT_TQCTL(0), reg);
1505
1506         ixl_enable_legacy(hw);
1507
1508         return;
1509 }
1510
1511
1512 /*********************************************************************
1513  *
1514  *  MSIX VSI Interrupt Service routine
1515  *
1516  **********************************************************************/
1517 void
1518 ixl_msix_que(void *arg)
1519 {
1520         struct ixl_queue        *que = arg;
1521         struct ixl_vsi  *vsi = que->vsi;
1522         struct i40e_hw  *hw = vsi->hw;
1523         struct tx_ring  *txr = &que->txr;
1524         bool            more_tx, more_rx;
1525
1526         /* Protect against spurious interrupts */
1527         if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1528                 return;
1529
1530         ++que->irqs;
1531
1532         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1533
1534         IXL_TX_LOCK(txr);
1535         more_tx = ixl_txeof(que);
1536         /*
1537         ** Make certain that if the stack
1538         ** has anything queued the task gets
1539         ** scheduled to handle it.
1540         */
1541         if (!drbr_empty(vsi->ifp, txr->br))
1542                 more_tx = 1;
1543         IXL_TX_UNLOCK(txr);
1544
1545         ixl_set_queue_rx_itr(que);
1546         ixl_set_queue_tx_itr(que);
1547
1548         if (more_tx || more_rx)
1549                 taskqueue_enqueue(que->tq, &que->task);
1550         else
1551                 ixl_enable_queue(hw, que->me);
1552
1553         return;
1554 }
1555
1556
1557 /*********************************************************************
1558  *
1559  *  MSIX Admin Queue Interrupt Service routine
1560  *
1561  **********************************************************************/
1562 static void
1563 ixl_msix_adminq(void *arg)
1564 {
1565         struct ixl_pf   *pf = arg;
1566         struct i40e_hw  *hw = &pf->hw;
1567         u32             reg, mask, rstat_reg;
1568         bool            do_task = FALSE;
1569
1570         ++pf->admin_irq;
1571
1572         reg = rd32(hw, I40E_PFINT_ICR0);
1573         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1574
1575         /* Check on the cause */
1576         if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) {
1577                 mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK;
1578                 do_task = TRUE;
1579         }
1580
1581         if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1582                 ixl_handle_mdd_event(pf);
1583                 mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK;
1584         }
1585
1586         if (reg & I40E_PFINT_ICR0_GRST_MASK) {
1587                 device_printf(pf->dev, "Reset Requested!\n");
1588                 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT);
1589                 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
1590                     >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
1591                 device_printf(pf->dev, "Reset type: ");
1592                 switch (rstat_reg) {
1593                 /* These others might be handled similarly to an EMPR reset */
1594                 case I40E_RESET_CORER:
1595                         printf("CORER\n");
1596                         break;
1597                 case I40E_RESET_GLOBR:
1598                         printf("GLOBR\n");
1599                         break;
1600                 case I40E_RESET_EMPR:
1601                         printf("EMPR\n");
1602                         atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
1603                         break;
1604                 default:
1605                         printf("?\n");
1606                         break;
1607                 }
1608                 // overload admin queue task to check reset progress?
1609                 do_task = TRUE;
1610         }
1611
1612         if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) {
1613                 device_printf(pf->dev, "ECC Error detected!\n");
1614         }
1615
1616         if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) {
1617                 device_printf(pf->dev, "HMC Error detected!\n");
1618         }
1619
1620         if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) {
1621                 device_printf(pf->dev, "PCI Exception detected!\n");
1622         }
1623
1624 #ifdef PCI_IOV
1625         if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1626                 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1627                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1628         }
1629 #endif
1630
1631         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1632         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1633         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1634
1635         if (do_task)
1636                 taskqueue_enqueue(pf->tq, &pf->adminq);
1637 }
1638
1639 /*********************************************************************
1640  *
1641  *  Media Ioctl callback
1642  *
1643  *  This routine is called whenever the user queries the status of
1644  *  the interface using ifconfig.
1645  *
1646  **********************************************************************/
1647 static void
1648 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1649 {
1650         struct ixl_vsi  *vsi = ifp->if_softc;
1651         struct ixl_pf   *pf = vsi->back;
1652         struct i40e_hw  *hw = &pf->hw;
1653
1654         INIT_DEBUGOUT("ixl_media_status: begin");
1655         IXL_PF_LOCK(pf);
1656
1657         hw->phy.get_link_info = TRUE;
1658         i40e_get_link_status(hw, &pf->link_up);
1659         ixl_update_link_status(pf);
1660
1661         ifmr->ifm_status = IFM_AVALID;
1662         ifmr->ifm_active = IFM_ETHER;
1663
1664         if (!pf->link_up) {
1665                 IXL_PF_UNLOCK(pf);
1666                 return;
1667         }
1668
1669         ifmr->ifm_status |= IFM_ACTIVE;
1670
1671         /* Hardware always does full-duplex */
1672         ifmr->ifm_active |= IFM_FDX;
1673
1674         switch (hw->phy.link_info.phy_type) {
1675                 /* 100 M */
1676                 case I40E_PHY_TYPE_100BASE_TX:
1677                         ifmr->ifm_active |= IFM_100_TX;
1678                         break;
1679                 /* 1 G */
1680                 case I40E_PHY_TYPE_1000BASE_T:
1681                         ifmr->ifm_active |= IFM_1000_T;
1682                         break;
1683                 case I40E_PHY_TYPE_1000BASE_SX:
1684                         ifmr->ifm_active |= IFM_1000_SX;
1685                         break;
1686                 case I40E_PHY_TYPE_1000BASE_LX:
1687                         ifmr->ifm_active |= IFM_1000_LX;
1688                         break;
1689                 case I40E_PHY_TYPE_1000BASE_T_OPTICAL:
1690                         ifmr->ifm_active |= IFM_OTHER;
1691                         break;
1692                 /* 10 G */
1693                 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1694                         ifmr->ifm_active |= IFM_10G_TWINAX;
1695                         break;
1696                 case I40E_PHY_TYPE_10GBASE_SR:
1697                         ifmr->ifm_active |= IFM_10G_SR;
1698                         break;
1699                 case I40E_PHY_TYPE_10GBASE_LR:
1700                         ifmr->ifm_active |= IFM_10G_LR;
1701                         break;
1702                 case I40E_PHY_TYPE_10GBASE_T:
1703                         ifmr->ifm_active |= IFM_10G_T;
1704                         break;
1705                 case I40E_PHY_TYPE_XAUI:
1706                 case I40E_PHY_TYPE_XFI:
1707                 case I40E_PHY_TYPE_10GBASE_AOC:
1708                         ifmr->ifm_active |= IFM_OTHER;
1709                         break;
1710                 /* 40 G */
1711                 case I40E_PHY_TYPE_40GBASE_CR4:
1712                 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1713                         ifmr->ifm_active |= IFM_40G_CR4;
1714                         break;
1715                 case I40E_PHY_TYPE_40GBASE_SR4:
1716                         ifmr->ifm_active |= IFM_40G_SR4;
1717                         break;
1718                 case I40E_PHY_TYPE_40GBASE_LR4:
1719                         ifmr->ifm_active |= IFM_40G_LR4;
1720                         break;
1721                 case I40E_PHY_TYPE_XLAUI:
1722                         ifmr->ifm_active |= IFM_OTHER;
1723                         break;
1724 #ifndef IFM_ETH_XTYPE
1725                 case I40E_PHY_TYPE_1000BASE_KX:
1726                         ifmr->ifm_active |= IFM_1000_CX;
1727                         break;
1728                 case I40E_PHY_TYPE_SGMII:
1729                         ifmr->ifm_active |= IFM_OTHER;
1730                         break;
1731                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1732                 case I40E_PHY_TYPE_10GBASE_CR1:
1733                         ifmr->ifm_active |= IFM_10G_TWINAX;
1734                         break;
1735                 case I40E_PHY_TYPE_10GBASE_KX4:
1736                         ifmr->ifm_active |= IFM_10G_CX4;
1737                         break;
1738                 case I40E_PHY_TYPE_10GBASE_KR:
1739                         ifmr->ifm_active |= IFM_10G_SR;
1740                         break;
1741                 case I40E_PHY_TYPE_SFI:
1742                         ifmr->ifm_active |= IFM_OTHER;
1743                         break;
1744                 case I40E_PHY_TYPE_40GBASE_KR4:
1745                 case I40E_PHY_TYPE_XLPPI:
1746                 case I40E_PHY_TYPE_40GBASE_AOC:
1747                         ifmr->ifm_active |= IFM_40G_SR4;
1748                         break;
1749 #else
1750                 case I40E_PHY_TYPE_1000BASE_KX:
1751                         ifmr->ifm_active |= IFM_1000_KX;
1752                         break;
1753                 case I40E_PHY_TYPE_SGMII:
1754                         ifmr->ifm_active |= IFM_1000_SGMII;
1755                         break;
1756                 /* ERJ: What's the difference between these? */
1757                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1758                 case I40E_PHY_TYPE_10GBASE_CR1:
1759                         ifmr->ifm_active |= IFM_10G_CR1;
1760                         break;
1761                 case I40E_PHY_TYPE_10GBASE_KX4:
1762                         ifmr->ifm_active |= IFM_10G_KX4;
1763                         break;
1764                 case I40E_PHY_TYPE_10GBASE_KR:
1765                         ifmr->ifm_active |= IFM_10G_KR;
1766                         break;
1767                 case I40E_PHY_TYPE_SFI:
1768                         ifmr->ifm_active |= IFM_10G_SFI;
1769                         break;
1770                 /* Our single 20G media type */
1771                 case I40E_PHY_TYPE_20GBASE_KR2:
1772                         ifmr->ifm_active |= IFM_20G_KR2;
1773                         break;
1774                 case I40E_PHY_TYPE_40GBASE_KR4:
1775                         ifmr->ifm_active |= IFM_40G_KR4;
1776                         break;
1777                 case I40E_PHY_TYPE_XLPPI:
1778                 case I40E_PHY_TYPE_40GBASE_AOC:
1779                         ifmr->ifm_active |= IFM_40G_XLPPI;
1780                         break;
1781 #endif
1782                 /* Unknown to driver */
1783                 default:
1784                         ifmr->ifm_active |= IFM_UNKNOWN;
1785                         break;
1786         }
1787         /* Report flow control status as well */
1788         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1789                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1790         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1791                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1792
1793         IXL_PF_UNLOCK(pf);
1794
1795         return;
1796 }
1797
1798 /*
1799  * NOTE: Fortville does not support forcing media speeds. Instead,
1800  * use the set_advertise sysctl to set the speeds Fortville
1801  * will advertise or be allowed to operate at.
1802  */
1803 static int
1804 ixl_media_change(struct ifnet * ifp)
1805 {
1806         struct ixl_vsi *vsi = ifp->if_softc;
1807         struct ifmedia *ifm = &vsi->media;
1808
1809         INIT_DEBUGOUT("ixl_media_change: begin");
1810
1811         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1812                 return (EINVAL);
1813
1814         if_printf(ifp, "Media change is not supported.\n");
1815
1816         return (ENODEV);
1817 }
1818
1819
1820 #ifdef IXL_FDIR
1821 /*
1822 ** ATR: Application Targetted Receive - creates a filter
1823 **      based on TX flow info that will keep the receive
1824 **      portion of the flow on the same queue. Based on the
1825 **      implementation this is only available for TCP connections
1826 */
1827 void
1828 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1829 {
1830         struct ixl_vsi                  *vsi = que->vsi;
1831         struct tx_ring                  *txr = &que->txr;
1832         struct i40e_filter_program_desc *FDIR;
1833         u32                             ptype, dtype;
1834         int                             idx;
1835
1836         /* check if ATR is enabled and sample rate */
1837         if ((!ixl_enable_fdir) || (!txr->atr_rate))
1838                 return;
1839         /*
1840         ** We sample all TCP SYN/FIN packets,
1841         ** or at the selected sample rate
1842         */
1843         txr->atr_count++;
1844         if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1845             (txr->atr_count < txr->atr_rate))
1846                 return;
1847         txr->atr_count = 0;
1848
1849         /* Get a descriptor to use */
1850         idx = txr->next_avail;
1851         FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1852         if (++idx == que->num_desc)
1853                 idx = 0;
1854         txr->avail--;
1855         txr->next_avail = idx;
1856
1857         ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1858             I40E_TXD_FLTR_QW0_QINDEX_MASK;
1859
1860         ptype |= (etype == ETHERTYPE_IP) ?
1861             (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1862             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1863             (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1864             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1865
1866         ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1867
1868         dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1869
1870         /*
1871         ** We use the TCP TH_FIN as a trigger to remove
1872         ** the filter, otherwise its an update.
1873         */
1874         dtype |= (th->th_flags & TH_FIN) ?
1875             (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1876             I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1877             (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1878             I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1879
1880         dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1881             I40E_TXD_FLTR_QW1_DEST_SHIFT;
1882
1883         dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1884             I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1885
1886         FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1887         FDIR->dtype_cmd_cntindex = htole32(dtype);
1888         return;
1889 }
1890 #endif
1891
1892
1893 static void
1894 ixl_set_promisc(struct ixl_vsi *vsi)
1895 {
1896         struct ifnet    *ifp = vsi->ifp;
1897         struct i40e_hw  *hw = vsi->hw;
1898         int             err, mcnt = 0;
1899         bool            uni = FALSE, multi = FALSE;
1900
1901         if (ifp->if_flags & IFF_ALLMULTI)
1902                 multi = TRUE;
1903         else { /* Need to count the multicast addresses */
1904                 struct  ifmultiaddr *ifma;
1905                 if_maddr_rlock(ifp);
1906                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1907                         if (ifma->ifma_addr->sa_family != AF_LINK)
1908                                 continue;
1909                         if (mcnt == MAX_MULTICAST_ADDR)
1910                                 break;
1911                         mcnt++;
1912                 }
1913                 if_maddr_runlock(ifp);
1914         }
1915
1916         if (mcnt >= MAX_MULTICAST_ADDR)
1917                 multi = TRUE;
1918         if (ifp->if_flags & IFF_PROMISC)
1919                 uni = TRUE;
1920
1921         err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1922             vsi->seid, uni, NULL);
1923         err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1924             vsi->seid, multi, NULL);
1925         return;
1926 }
1927
1928 /*********************************************************************
1929  *      Filter Routines
1930  *
1931  *      Routines for multicast and vlan filter management.
1932  *
1933  *********************************************************************/
1934 static void
1935 ixl_add_multi(struct ixl_vsi *vsi)
1936 {
1937         struct  ifmultiaddr     *ifma;
1938         struct ifnet            *ifp = vsi->ifp;
1939         struct i40e_hw          *hw = vsi->hw;
1940         int                     mcnt = 0, flags;
1941
1942         IOCTL_DEBUGOUT("ixl_add_multi: begin");
1943
1944         if_maddr_rlock(ifp);
1945         /*
1946         ** First just get a count, to decide if we
1947         ** we simply use multicast promiscuous.
1948         */
1949         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1950                 if (ifma->ifma_addr->sa_family != AF_LINK)
1951                         continue;
1952                 mcnt++;
1953         }
1954         if_maddr_runlock(ifp);
1955
1956         if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1957                 /* delete existing MC filters */
1958                 ixl_del_hw_filters(vsi, mcnt);
1959                 i40e_aq_set_vsi_multicast_promiscuous(hw,
1960                     vsi->seid, TRUE, NULL);
1961                 return;
1962         }
1963
1964         mcnt = 0;
1965         if_maddr_rlock(ifp);
1966         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1967                 if (ifma->ifma_addr->sa_family != AF_LINK)
1968                         continue;
1969                 ixl_add_mc_filter(vsi,
1970                     (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1971                 mcnt++;
1972         }
1973         if_maddr_runlock(ifp);
1974         if (mcnt > 0) {
1975                 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1976                 ixl_add_hw_filters(vsi, flags, mcnt);
1977         }
1978
1979         IOCTL_DEBUGOUT("ixl_add_multi: end");
1980         return;
1981 }
1982
1983 static void
1984 ixl_del_multi(struct ixl_vsi *vsi)
1985 {
1986         struct ifnet            *ifp = vsi->ifp;
1987         struct ifmultiaddr      *ifma;
1988         struct ixl_mac_filter   *f;
1989         int                     mcnt = 0;
1990         bool            match = FALSE;
1991
1992         IOCTL_DEBUGOUT("ixl_del_multi: begin");
1993
1994         /* Search for removed multicast addresses */
1995         if_maddr_rlock(ifp);
1996         SLIST_FOREACH(f, &vsi->ftl, next) {
1997                 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1998                         match = FALSE;
1999                         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2000                                 if (ifma->ifma_addr->sa_family != AF_LINK)
2001                                         continue;
2002                                 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2003                                 if (cmp_etheraddr(f->macaddr, mc_addr)) {
2004                                         match = TRUE;
2005                                         break;
2006                                 }
2007                         }
2008                         if (match == FALSE) {
2009                                 f->flags |= IXL_FILTER_DEL;
2010                                 mcnt++;
2011                         }
2012                 }
2013         }
2014         if_maddr_runlock(ifp);
2015
2016         if (mcnt > 0)
2017                 ixl_del_hw_filters(vsi, mcnt);
2018 }
2019
2020
2021 /*********************************************************************
2022  *  Timer routine
2023  *
2024  *  This routine checks for link status,updates statistics,
2025  *  and runs the watchdog check.
2026  *
2027  *  Only runs when the driver is configured UP and RUNNING.
2028  *
2029  **********************************************************************/
2030
2031 static void
2032 ixl_local_timer(void *arg)
2033 {
2034         struct ixl_pf           *pf = arg;
2035         struct i40e_hw          *hw = &pf->hw;
2036         struct ixl_vsi          *vsi = &pf->vsi;
2037         struct ixl_queue        *que = vsi->queues;
2038         device_t                dev = pf->dev;
2039         int                     hung = 0;
2040         u32                     mask;
2041
2042         mtx_assert(&pf->pf_mtx, MA_OWNED);
2043
2044         /* Fire off the adminq task */
2045         taskqueue_enqueue(pf->tq, &pf->adminq);
2046
2047         /* Update stats */
2048         ixl_update_stats_counters(pf);
2049
2050         /*
2051         ** Check status of the queues
2052         */
2053         mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
2054                 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
2055
2056         for (int i = 0; i < vsi->num_queues; i++, que++) {
2057                 /* Any queues with outstanding work get a sw irq */
2058                 if (que->busy)
2059                         wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
2060                 /*
2061                 ** Each time txeof runs without cleaning, but there
2062                 ** are uncleaned descriptors it increments busy. If
2063                 ** we get to 5 we declare it hung.
2064                 */
2065                 if (que->busy == IXL_QUEUE_HUNG) {
2066                         ++hung;
2067                         /* Mark the queue as inactive */
2068                         vsi->active_queues &= ~((u64)1 << que->me);
2069                         continue;
2070                 } else {
2071                         /* Check if we've come back from hung */
2072                         if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
2073                                 vsi->active_queues |= ((u64)1 << que->me);
2074                 }
2075                 if (que->busy >= IXL_MAX_TX_BUSY) {
2076 #ifdef IXL_DEBUG
2077                         device_printf(dev,"Warning queue %d "
2078                             "appears to be hung!\n", i);
2079 #endif
2080                         que->busy = IXL_QUEUE_HUNG;
2081                         ++hung;
2082                 }
2083         }
2084         /* Only reinit if all queues show hung */
2085         if (hung == vsi->num_queues)
2086                 goto hung;
2087
2088         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
2089         return;
2090
2091 hung:
2092         device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
2093         ixl_init_locked(pf);
2094 }
2095
2096 /*
2097 ** Note: this routine updates the OS on the link state
2098 **      the real check of the hardware only happens with
2099 **      a link interrupt.
2100 */
2101 static void
2102 ixl_update_link_status(struct ixl_pf *pf)
2103 {
2104         struct ixl_vsi          *vsi = &pf->vsi;
2105         struct i40e_hw          *hw = &pf->hw;
2106         struct ifnet            *ifp = vsi->ifp;
2107         device_t                dev = pf->dev;
2108
2109         if (pf->link_up) {
2110                 if (vsi->link_active == FALSE) {
2111                         pf->fc = hw->fc.current_mode;
2112                         if (bootverbose) {
2113                                 device_printf(dev,"Link is up %d Gbps %s,"
2114                                     " Flow Control: %s\n",
2115                                     ((pf->link_speed ==
2116                                     I40E_LINK_SPEED_40GB)? 40:10),
2117                                     "Full Duplex", ixl_fc_string[pf->fc]);
2118                         }
2119                         vsi->link_active = TRUE;
2120                         /*
2121                         ** Warn user if link speed on NPAR enabled
2122                         ** partition is not at least 10GB
2123                         */
2124                         if (hw->func_caps.npar_enable &&
2125                            (hw->phy.link_info.link_speed ==
2126                            I40E_LINK_SPEED_1GB ||
2127                            hw->phy.link_info.link_speed ==
2128                            I40E_LINK_SPEED_100MB))
2129                                 device_printf(dev, "The partition detected"
2130                                     "link speed that is less than 10Gbps\n");
2131                         if_link_state_change(ifp, LINK_STATE_UP);
2132                 }
2133         } else { /* Link down */
2134                 if (vsi->link_active == TRUE) {
2135                         if (bootverbose)
2136                                 device_printf(dev, "Link is Down\n");
2137                         if_link_state_change(ifp, LINK_STATE_DOWN);
2138                         vsi->link_active = FALSE;
2139                 }
2140         }
2141
2142         return;
2143 }
2144
2145 static void
2146 ixl_stop(struct ixl_pf *pf)
2147 {
2148         IXL_PF_LOCK(pf);
2149         ixl_stop_locked(pf);
2150         IXL_PF_UNLOCK(pf);
2151
2152         ixl_teardown_queue_msix(&pf->vsi);
2153         ixl_free_queue_tqs(&pf->vsi);
2154 }
2155
2156 /*********************************************************************
2157  *
2158  *  This routine disables all traffic on the adapter by issuing a
2159  *  global reset on the MAC and deallocates TX/RX buffers.
2160  *
2161  **********************************************************************/
2162
2163 static void
2164 ixl_stop_locked(struct ixl_pf *pf)
2165 {
2166         struct ixl_vsi  *vsi = &pf->vsi;
2167         struct ifnet    *ifp = vsi->ifp;
2168
2169         INIT_DEBUGOUT("ixl_stop: begin\n");
2170
2171         IXL_PF_LOCK_ASSERT(pf);
2172
2173         /* Stop the local timer */
2174         callout_stop(&pf->timer);
2175
2176         ixl_disable_rings_intr(vsi);
2177         ixl_disable_rings(vsi);
2178
2179         /* Tell the stack that the interface is no longer active */
2180         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING);
2181 }
2182
2183
2184 /*********************************************************************
2185  *
2186  *  Setup MSIX Interrupt resources and handlers for the VSI
2187  *
2188  **********************************************************************/
2189 static int
2190 ixl_assign_vsi_legacy(struct ixl_pf *pf)
2191 {
2192         device_t        dev = pf->dev;
2193         struct          ixl_vsi *vsi = &pf->vsi;
2194         struct          ixl_queue *que = vsi->queues;
2195         int             error, rid = 0;
2196
2197         if (pf->msix == 1)
2198                 rid = 1;
2199         pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2200             &rid, RF_SHAREABLE | RF_ACTIVE);
2201         if (pf->res == NULL) {
2202                 device_printf(dev, "Unable to allocate"
2203                     " bus resource: vsi legacy/msi interrupt\n");
2204                 return (ENXIO);
2205         }
2206
2207         /* Set the handler function */
2208         error = bus_setup_intr(dev, pf->res,
2209             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2210             ixl_intr, pf, &pf->tag);
2211         if (error) {
2212                 pf->res = NULL;
2213                 device_printf(dev, "Failed to register legacy/msi handler\n");
2214                 return (error);
2215         }
2216         bus_describe_intr(dev, pf->res, pf->tag, "irq0");
2217         TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2218         TASK_INIT(&que->task, 0, ixl_handle_que, que);
2219         que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2220             taskqueue_thread_enqueue, &que->tq);
2221         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2222             device_get_nameunit(dev));
2223         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2224
2225         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2226             taskqueue_thread_enqueue, &pf->tq);
2227         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2228             device_get_nameunit(dev));
2229
2230         return (0);
2231 }
2232
2233 static int
2234 ixl_setup_adminq_tq(struct ixl_pf *pf)
2235 {
2236         device_t dev = pf->dev;
2237         int error = 0;
2238
2239         /* Tasklet for Admin Queue interrupts */
2240         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2241 #ifdef PCI_IOV
2242         /* VFLR Tasklet */
2243         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2244 #endif
2245         /* Create and start Admin Queue taskqueue */
2246         pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT,
2247             taskqueue_thread_enqueue, &pf->tq);
2248         if (!pf->tq) {
2249                 device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n");
2250                 return (ENOMEM);
2251         }
2252         error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq",
2253             device_get_nameunit(dev));
2254         if (error) {
2255                 device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n",
2256                     error);
2257                 taskqueue_free(pf->tq);
2258                 return (error);
2259         }
2260         return (0);
2261 }
2262
2263 static int
2264 ixl_setup_queue_tqs(struct ixl_vsi *vsi)
2265 {
2266         struct ixl_queue *que = vsi->queues;
2267         device_t dev = vsi->dev;
2268 #ifdef  RSS
2269         cpuset_t cpu_mask;
2270         int cpu_id;
2271 #endif
2272
2273         /* Create queue tasks and start queue taskqueues */
2274         for (int i = 0; i < vsi->num_queues; i++, que++) {
2275                 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2276                 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2277                 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2278                     taskqueue_thread_enqueue, &que->tq);
2279 #ifdef RSS
2280                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2281                 CPU_SETOF(cpu_id, &cpu_mask);
2282                 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2283                     &cpu_mask, "%s (bucket %d)",
2284                     device_get_nameunit(dev), cpu_id);
2285 #else
2286                 taskqueue_start_threads(&que->tq, 1, PI_NET,
2287                     "%s (que %d)", device_get_nameunit(dev), que->me);
2288 #endif
2289         }
2290
2291         return (0);
2292 }
2293
2294 static void
2295 ixl_free_adminq_tq(struct ixl_pf *pf)
2296 {
2297         if (pf->tq) {
2298                 taskqueue_free(pf->tq);
2299                 pf->tq = NULL;
2300         }
2301 }
2302
2303 static void
2304 ixl_free_queue_tqs(struct ixl_vsi *vsi)
2305 {
2306         struct ixl_queue *que = vsi->queues;
2307
2308         for (int i = 0; i < vsi->num_queues; i++, que++) {
2309                 if (que->tq) {
2310                         taskqueue_free(que->tq);
2311                         que->tq = NULL;
2312                 }
2313         }
2314 }
2315
2316 static int
2317 ixl_setup_adminq_msix(struct ixl_pf *pf)
2318 {
2319         device_t dev = pf->dev;
2320         int rid, error = 0;
2321
2322         /* Admin IRQ rid is 1, vector is 0 */
2323         rid = 1;
2324         /* Get interrupt resource from bus */
2325         pf->res = bus_alloc_resource_any(dev,
2326             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2327         if (!pf->res) {
2328                 device_printf(dev, "bus_alloc_resource_any() for Admin Queue"
2329                     " interrupt failed [rid=%d]\n", rid);
2330                 return (ENXIO);
2331         }
2332         /* Then associate interrupt with handler */
2333         error = bus_setup_intr(dev, pf->res,
2334             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2335             ixl_msix_adminq, pf, &pf->tag);
2336         if (error) {
2337                 pf->res = NULL;
2338                 device_printf(dev, "bus_setup_intr() for Admin Queue"
2339                     " interrupt handler failed, error %d\n", error);
2340                 return (ENXIO);
2341         }
2342         error = bus_describe_intr(dev, pf->res, pf->tag, "aq");
2343         if (error) {
2344                 /* Probably non-fatal? */
2345                 device_printf(dev, "bus_describe_intr() for Admin Queue"
2346                     " interrupt name failed, error %d\n", error);
2347         }
2348         pf->admvec = 0;
2349
2350         return (0);
2351 }
2352
2353 /*
2354  * Allocate interrupt resources from bus and associate an interrupt handler
2355  * to those for the VSI's queues.
2356  */
2357 static int
2358 ixl_setup_queue_msix(struct ixl_vsi *vsi)
2359 {
2360         device_t        dev = vsi->dev;
2361         struct          ixl_queue *que = vsi->queues;
2362         struct          tx_ring  *txr;
2363         int             error, rid, vector = 1;
2364
2365         /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */
2366         for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2367                 int cpu_id = i;
2368                 rid = vector + 1;
2369                 txr = &que->txr;
2370                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2371                     RF_SHAREABLE | RF_ACTIVE);
2372                 if (!que->res) {
2373                         device_printf(dev, "bus_alloc_resource_any() for"
2374                             " Queue %d interrupt failed [rid=%d]\n",
2375                             que->me, rid);
2376                         return (ENXIO);
2377                 }
2378                 /* Set the handler function */
2379                 error = bus_setup_intr(dev, que->res,
2380                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2381                     ixl_msix_que, que, &que->tag);
2382                 if (error) {
2383                         device_printf(dev, "bus_setup_intr() for Queue %d"
2384                             " interrupt handler failed, error %d\n",
2385                             que->me, error);
2386                         // TODO: Check for error from this?
2387                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2388                         return (error);
2389                 }
2390                 error = bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2391                 if (error) {
2392                         device_printf(dev, "bus_describe_intr() for Queue %d"
2393                             " interrupt name failed, error %d\n",
2394                             que->me, error);
2395                 }
2396                 /* Bind the vector to a CPU */
2397 #ifdef RSS
2398                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2399 #endif
2400                 error = bus_bind_intr(dev, que->res, cpu_id);
2401                 if (error) {
2402                         device_printf(dev, "bus_bind_intr() for Queue %d"
2403                             " to CPU %d failed, error %d\n",
2404                             que->me, cpu_id, error);
2405                 }
2406                 que->msix = vector;
2407         }
2408
2409         return (0);
2410 }
2411
2412
2413 /*
2414  * Allocate MSI/X vectors
2415  */
2416 static int
2417 ixl_init_msix(struct ixl_pf *pf)
2418 {
2419         device_t dev = pf->dev;
2420         int rid, want, vectors, queues, available;
2421
2422         /* Override by tuneable */
2423         if (ixl_enable_msix == 0)
2424                 goto no_msix;
2425
2426         /*
2427         ** When used in a virtualized environment
2428         ** PCI BUSMASTER capability may not be set
2429         ** so explicity set it here and rewrite
2430         ** the ENABLE in the MSIX control register
2431         ** at this point to cause the host to
2432         ** successfully initialize us.
2433         */
2434         {
2435                 u16 pci_cmd_word;
2436                 int msix_ctrl;
2437                 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2438                 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2439                 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2440                 pci_find_cap(dev, PCIY_MSIX, &rid);
2441                 rid += PCIR_MSIX_CTRL;
2442                 msix_ctrl = pci_read_config(dev, rid, 2);
2443                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2444                 pci_write_config(dev, rid, msix_ctrl, 2);
2445         }
2446
2447         /* First try MSI/X */
2448         rid = PCIR_BAR(IXL_BAR);
2449         pf->msix_mem = bus_alloc_resource_any(dev,
2450             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2451         if (!pf->msix_mem) {
2452                 /* May not be enabled */
2453                 device_printf(pf->dev,
2454                     "Unable to map MSIX table\n");
2455                 goto no_msix;
2456         }
2457
2458         available = pci_msix_count(dev);
2459         if (available == 0) { /* system has msix disabled */
2460                 bus_release_resource(dev, SYS_RES_MEMORY,
2461                     rid, pf->msix_mem);
2462                 pf->msix_mem = NULL;
2463                 goto no_msix;
2464         }
2465
2466         /* Figure out a reasonable auto config value */
2467         queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2468
2469         /* Override with tunable value if tunable is less than autoconfig count */
2470         if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2471                 queues = ixl_max_queues;
2472         else if ((ixl_max_queues != 0) && (ixl_max_queues > queues))
2473                 device_printf(dev, "ixl_max_queues > # of cpus, using "
2474                     "autoconfig amount...\n");
2475         /* Or limit maximum auto-configured queues to 8 */
2476         else if ((ixl_max_queues == 0) && (queues > 8))
2477                 queues = 8;
2478
2479 #ifdef  RSS
2480         /* If we're doing RSS, clamp at the number of RSS buckets */
2481         if (queues > rss_getnumbuckets())
2482                 queues = rss_getnumbuckets();
2483 #endif
2484
2485         /*
2486         ** Want one vector (RX/TX pair) per queue
2487         ** plus an additional for the admin queue.
2488         */
2489         want = queues + 1;
2490         if (want <= available)  /* Have enough */
2491                 vectors = want;
2492         else {
2493                 device_printf(pf->dev,
2494                     "MSIX Configuration Problem, "
2495                     "%d vectors available but %d wanted!\n",
2496                     available, want);
2497                 return (0); /* Will go to Legacy setup */
2498         }
2499
2500         if (pci_alloc_msix(dev, &vectors) == 0) {
2501                 device_printf(pf->dev,
2502                     "Using MSIX interrupts with %d vectors\n", vectors);
2503                 pf->msix = vectors;
2504                 pf->vsi.num_queues = queues;
2505 #ifdef RSS
2506                 /*
2507                  * If we're doing RSS, the number of queues needs to
2508                  * match the number of RSS buckets that are configured.
2509                  *
2510                  * + If there's more queues than RSS buckets, we'll end
2511                  *   up with queues that get no traffic.
2512                  *
2513                  * + If there's more RSS buckets than queues, we'll end
2514                  *   up having multiple RSS buckets map to the same queue,
2515                  *   so there'll be some contention.
2516                  */
2517                 if (queues != rss_getnumbuckets()) {
2518                         device_printf(dev,
2519                             "%s: queues (%d) != RSS buckets (%d)"
2520                             "; performance will be impacted.\n",
2521                             __func__, queues, rss_getnumbuckets());
2522                 }
2523 #endif
2524                 return (vectors);
2525         }
2526 no_msix:
2527         vectors = pci_msi_count(dev);
2528         pf->vsi.num_queues = 1;
2529         ixl_max_queues = 1;
2530         ixl_enable_msix = 0;
2531         if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2532                 device_printf(pf->dev, "Using an MSI interrupt\n");
2533         else {
2534                 vectors = 0;
2535                 device_printf(pf->dev, "Using a Legacy interrupt\n");
2536         }
2537         return (vectors);
2538 }
2539
2540 /*
2541  * Configure admin queue/misc interrupt cause registers in hardware.
2542  */
2543 static void
2544 ixl_configure_intr0_msix(struct ixl_pf *pf)
2545 {
2546         struct i40e_hw *hw = &pf->hw;
2547         u32 reg;
2548
2549         /* First set up the adminq - vector 0 */
2550         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2551         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2552
2553         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2554             I40E_PFINT_ICR0_ENA_GRST_MASK |
2555             I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
2556             I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2557             I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2558             I40E_PFINT_ICR0_ENA_VFLR_MASK |
2559             I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2560         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2561
2562         /*
2563          * 0x7FF is the end of the queue list.
2564          * This means we won't use MSI-X vector 0 for a queue interrupt
2565          * in MSIX mode.
2566          */
2567         wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2568         /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */
2569         wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E);
2570
2571         wr32(hw, I40E_PFINT_DYN_CTL0,
2572             I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2573             I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2574
2575         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2576 }
2577
2578 /*
2579  * Configure queue interrupt cause registers in hardware.
2580  */
2581 static void
2582 ixl_configure_queue_intr_msix(struct ixl_pf *pf)
2583 {
2584         struct i40e_hw  *hw = &pf->hw;
2585         struct ixl_vsi *vsi = &pf->vsi;
2586         u32             reg;
2587         u16             vector = 1;
2588
2589         for (int i = 0; i < vsi->num_queues; i++, vector++) {
2590                 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
2591                 /* First queue type is RX / 0 */
2592                 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2593
2594                 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2595                 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2596                 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2597                 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2598                 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2599                 wr32(hw, I40E_QINT_RQCTL(i), reg);
2600
2601                 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2602                 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2603                 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2604                 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2605                 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2606                 wr32(hw, I40E_QINT_TQCTL(i), reg);
2607         }
2608 }
2609
2610 /*
2611  * Configure for MSI single vector operation
2612  */
2613 static void
2614 ixl_configure_legacy(struct ixl_pf *pf)
2615 {
2616         struct i40e_hw  *hw = &pf->hw;
2617         u32             reg;
2618
2619         wr32(hw, I40E_PFINT_ITR0(0), 0);
2620         wr32(hw, I40E_PFINT_ITR0(1), 0);
2621
2622         /* Setup "other" causes */
2623         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2624             | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2625             | I40E_PFINT_ICR0_ENA_GRST_MASK
2626             | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2627             | I40E_PFINT_ICR0_ENA_GPIO_MASK
2628             | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2629             | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2630             | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2631             | I40E_PFINT_ICR0_ENA_VFLR_MASK
2632             | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2633             ;
2634         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2635
2636         /* SW_ITR_IDX = 0, but don't change INTENA */
2637         wr32(hw, I40E_PFINT_DYN_CTL0,
2638             I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2639             I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2640         /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2641         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2642
2643         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2644         wr32(hw, I40E_PFINT_LNKLST0, 0);
2645
2646         /* Associate the queue pair to the vector and enable the q int */
2647         reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2648             | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2649             | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2650         wr32(hw, I40E_QINT_RQCTL(0), reg);
2651
2652         reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2653             | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2654             | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2655         wr32(hw, I40E_QINT_TQCTL(0), reg);
2656 }
2657
2658
2659 /*
2660  * Get initial ITR values from tunable values.
2661  */
2662 static void
2663 ixl_configure_itr(struct ixl_pf *pf)
2664 {
2665         struct i40e_hw          *hw = &pf->hw;
2666         struct ixl_vsi          *vsi = &pf->vsi;
2667         struct ixl_queue        *que = vsi->queues;
2668
2669         vsi->rx_itr_setting = ixl_rx_itr;
2670         vsi->tx_itr_setting = ixl_tx_itr;
2671
2672         for (int i = 0; i < vsi->num_queues; i++, que++) {
2673                 struct tx_ring  *txr = &que->txr;
2674                 struct rx_ring  *rxr = &que->rxr;
2675
2676                 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2677                     vsi->rx_itr_setting);
2678                 rxr->itr = vsi->rx_itr_setting;
2679                 rxr->latency = IXL_AVE_LATENCY;
2680
2681                 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2682                     vsi->tx_itr_setting);
2683                 txr->itr = vsi->tx_itr_setting;
2684                 txr->latency = IXL_AVE_LATENCY;
2685         }
2686 }
2687
2688
2689 static int
2690 ixl_allocate_pci_resources(struct ixl_pf *pf)
2691 {
2692         int             rid;
2693         device_t        dev = pf->dev;
2694
2695         rid = PCIR_BAR(0);
2696         pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2697             &rid, RF_ACTIVE);
2698
2699         if (!(pf->pci_mem)) {
2700                 device_printf(dev, "Unable to allocate bus resource: PCI memory\n");
2701                 return (ENXIO);
2702         }
2703
2704         pf->osdep.mem_bus_space_tag =
2705                 rman_get_bustag(pf->pci_mem);
2706         pf->osdep.mem_bus_space_handle =
2707                 rman_get_bushandle(pf->pci_mem);
2708         pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2709         pf->osdep.flush_reg = I40E_GLGEN_STAT;
2710         pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2711
2712         pf->hw.back = &pf->osdep;
2713
2714         /*
2715         ** Now setup MSI or MSI/X, should
2716         ** return us the number of supported
2717         ** vectors. (Will be 1 for MSI)
2718         */
2719         pf->msix = ixl_init_msix(pf);
2720         return (0);
2721 }
2722
2723 /*
2724  * Teardown and release the admin queue/misc vector
2725  * interrupt.
2726  */
2727 static int
2728 ixl_teardown_adminq_msix(struct ixl_pf *pf)
2729 {
2730         device_t                dev = pf->dev;
2731         int                     rid;
2732
2733         if (pf->admvec) /* we are doing MSIX */
2734                 rid = pf->admvec + 1;
2735         else
2736                 (pf->msix != 0) ? (rid = 1):(rid = 0);
2737
2738         // TODO: Check for errors from bus_teardown_intr
2739         // TODO: Check for errors from bus_release_resource
2740         if (pf->tag != NULL) {
2741                 bus_teardown_intr(dev, pf->res, pf->tag);
2742                 pf->tag = NULL;
2743         }
2744         if (pf->res != NULL) {
2745                 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2746                 pf->res = NULL;
2747         }
2748
2749         return (0);
2750 }
2751
2752 static int
2753 ixl_teardown_queue_msix(struct ixl_vsi *vsi)
2754 {
2755         struct ixl_queue        *que = vsi->queues;
2756         device_t                dev = vsi->dev;
2757         int                     rid, error = 0;
2758
2759         /* We may get here before stations are setup */
2760         if ((!ixl_enable_msix) || (que == NULL))
2761                 return (0);
2762
2763         /* Release all MSIX queue resources */
2764         for (int i = 0; i < vsi->num_queues; i++, que++) {
2765                 rid = que->msix + 1;
2766                 if (que->tag != NULL) {
2767                         error = bus_teardown_intr(dev, que->res, que->tag);
2768                         if (error) {
2769                                 device_printf(dev, "bus_teardown_intr() for"
2770                                     " Queue %d interrupt failed\n",
2771                                     que->me);
2772                                 // return (ENXIO);
2773                         }
2774                         que->tag = NULL;
2775                 }
2776                 if (que->res != NULL) {
2777                         error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2778                         if (error) {
2779                                 device_printf(dev, "bus_release_resource() for"
2780                                     " Queue %d interrupt failed [rid=%d]\n",
2781                                     que->me, rid);
2782                                 // return (ENXIO);
2783                         }
2784                         que->res = NULL;
2785                 }
2786         }
2787
2788         return (0);
2789 }
2790
2791 static void
2792 ixl_free_pci_resources(struct ixl_pf *pf)
2793 {
2794         device_t                dev = pf->dev;
2795         int                     memrid;
2796
2797         ixl_teardown_queue_msix(&pf->vsi);
2798         ixl_teardown_adminq_msix(pf);
2799
2800         if (pf->msix)
2801                 pci_release_msi(dev);
2802
2803         memrid = PCIR_BAR(IXL_BAR);
2804
2805         if (pf->msix_mem != NULL)
2806                 bus_release_resource(dev, SYS_RES_MEMORY,
2807                     memrid, pf->msix_mem);
2808
2809         if (pf->pci_mem != NULL)
2810                 bus_release_resource(dev, SYS_RES_MEMORY,
2811                     PCIR_BAR(0), pf->pci_mem);
2812
2813         return;
2814 }
2815
2816 static void
2817 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2818 {
2819         /* Display supported media types */
2820         if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2821                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2822
2823         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2824                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2825         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2826                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2827         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2828                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2829
2830         if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2831             phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2832             phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2833                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2834
2835         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2836                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2837         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2838                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2839         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2840                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2841
2842         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2843             phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2844             phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2845             phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2846             phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2847                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2848         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2849                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2850         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2851                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2852
2853 #ifndef IFM_ETH_XTYPE
2854         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2855                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2856
2857         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2858             phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2859             phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2860             phy_type & (1 << I40E_PHY_TYPE_SFI))
2861                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2862         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2863                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2864         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2865                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2866
2867         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2868                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2869         if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2870                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2871 #else
2872         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2873                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2874
2875         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2876             || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2877                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2878         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2879                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2880         if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2881                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2882         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2883                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2884         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2885                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2886
2887         if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2888                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2889
2890         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2891                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2892         if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2893                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2894 #endif
2895 }
2896
2897 /*********************************************************************
2898  *
2899  *  Setup networking device structure and register an interface.
2900  *
2901  **********************************************************************/
2902 static int
2903 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2904 {
2905         struct ifnet            *ifp;
2906         struct i40e_hw          *hw = vsi->hw;
2907         struct ixl_queue        *que = vsi->queues;
2908         struct i40e_aq_get_phy_abilities_resp abilities;
2909         enum i40e_status_code aq_error = 0;
2910
2911         INIT_DEBUGOUT("ixl_setup_interface: begin");
2912
2913         ifp = vsi->ifp = if_alloc(IFT_ETHER);
2914         if (ifp == NULL) {
2915                 device_printf(dev, "can not allocate ifnet structure\n");
2916                 return (-1);
2917         }
2918         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2919         ifp->if_mtu = ETHERMTU;
2920         ifp->if_baudrate = IF_Gbps(40);
2921         ifp->if_init = ixl_init;
2922         ifp->if_softc = vsi;
2923         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2924         ifp->if_ioctl = ixl_ioctl;
2925
2926 #if __FreeBSD_version >= 1100036
2927         if_setgetcounterfn(ifp, ixl_get_counter);
2928 #endif
2929
2930         ifp->if_transmit = ixl_mq_start;
2931
2932         ifp->if_qflush = ixl_qflush;
2933
2934         ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2935
2936         vsi->max_frame_size =
2937             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2938             + ETHER_VLAN_ENCAP_LEN;
2939
2940         /*
2941          * Tell the upper layer(s) we support long frames.
2942          */
2943         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2944
2945         ifp->if_capabilities |= IFCAP_HWCSUM;
2946         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2947         ifp->if_capabilities |= IFCAP_TSO;
2948         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2949         ifp->if_capabilities |= IFCAP_LRO;
2950
2951         /* VLAN capabilties */
2952         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2953                              |  IFCAP_VLAN_HWTSO
2954                              |  IFCAP_VLAN_MTU
2955                              |  IFCAP_VLAN_HWCSUM;
2956         ifp->if_capenable = ifp->if_capabilities;
2957
2958         /*
2959         ** Don't turn this on by default, if vlans are
2960         ** created on another pseudo device (eg. lagg)
2961         ** then vlan events are not passed thru, breaking
2962         ** operation, but with HW FILTER off it works. If
2963         ** using vlans directly on the ixl driver you can
2964         ** enable this and get full hardware tag filtering.
2965         */
2966         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2967
2968         /*
2969          * Specify the media types supported by this adapter and register
2970          * callbacks to update media and link information
2971          */
2972         ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2973                      ixl_media_status);
2974
2975         aq_error = i40e_aq_get_phy_capabilities(hw,
2976             FALSE, TRUE, &abilities, NULL);
2977         /* May need delay to detect fiber correctly */
2978         if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2979                 i40e_msec_delay(200);
2980                 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2981                     TRUE, &abilities, NULL);
2982         }
2983         if (aq_error) {
2984                 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2985                         device_printf(dev, "Unknown PHY type detected!\n");
2986                 else
2987                         device_printf(dev,
2988                             "Error getting supported media types, err %d,"
2989                             " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2990                 return (0);
2991         }
2992
2993         ixl_add_ifmedia(vsi, abilities.phy_type);
2994
2995         /* Use autoselect media by default */
2996         ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2997         ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2998
2999         ether_ifattach(ifp, hw->mac.addr);
3000
3001         return (0);
3002 }
3003
3004 /*
3005 ** Run when the Admin Queue gets a link state change interrupt.
3006 */
3007 static void
3008 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
3009 {
3010         struct i40e_hw  *hw = &pf->hw;
3011         device_t dev = pf->dev;
3012         struct i40e_aqc_get_link_status *status =
3013             (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
3014
3015         /* Request link status from adapter */
3016         hw->phy.get_link_info = TRUE;
3017         i40e_get_link_status(hw, &pf->link_up);
3018
3019         /* Print out message if an unqualified module is found */
3020         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
3021             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
3022             (!(status->link_info & I40E_AQ_LINK_UP)))
3023                 device_printf(dev, "Link failed because "
3024                     "an unqualified module was detected!\n");
3025
3026         /* Update OS link info */
3027         ixl_update_link_status(pf);
3028 }
3029
3030 /*********************************************************************
3031  *
3032  *  Get Firmware Switch configuration
3033  *      - this will need to be more robust when more complex
3034  *        switch configurations are enabled.
3035  *
3036  **********************************************************************/
3037 static int
3038 ixl_switch_config(struct ixl_pf *pf)
3039 {
3040         struct i40e_hw  *hw = &pf->hw;
3041         struct ixl_vsi  *vsi = &pf->vsi;
3042         device_t        dev = vsi->dev;
3043         struct i40e_aqc_get_switch_config_resp *sw_config;
3044         u8      aq_buf[I40E_AQ_LARGE_BUF];
3045         int     ret;
3046         u16     next = 0;
3047
3048         memset(&aq_buf, 0, sizeof(aq_buf));
3049         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
3050         ret = i40e_aq_get_switch_config(hw, sw_config,
3051             sizeof(aq_buf), &next, NULL);
3052         if (ret) {
3053                 device_printf(dev, "aq_get_switch_config() failed, error %d,"
3054                     " aq_error %d\n", ret, pf->hw.aq.asq_last_status);
3055                 return (ret);
3056         }
3057 #ifdef IXL_DEBUG
3058         device_printf(dev,
3059             "Switch config: header reported: %d in structure, %d total\n",
3060             sw_config->header.num_reported, sw_config->header.num_total);
3061         for (int i = 0; i < sw_config->header.num_reported; i++) {
3062                 device_printf(dev,
3063                     "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
3064                     sw_config->element[i].element_type,
3065                     sw_config->element[i].seid,
3066                     sw_config->element[i].uplink_seid,
3067                     sw_config->element[i].downlink_seid);
3068         }
3069 #endif
3070         /* Simplified due to a single VSI at the moment */
3071         vsi->uplink_seid = sw_config->element[0].uplink_seid;
3072         vsi->downlink_seid = sw_config->element[0].downlink_seid;
3073         vsi->seid = sw_config->element[0].seid;
3074         return (ret);
3075 }
3076
3077 /*********************************************************************
3078  *
3079  *  Initialize the VSI:  this handles contexts, which means things
3080  *                       like the number of descriptors, buffer size,
3081  *                       plus we init the rings thru this function.
3082  *
3083  **********************************************************************/
3084 static int
3085 ixl_initialize_vsi(struct ixl_vsi *vsi)
3086 {
3087         struct ixl_pf           *pf = vsi->back;
3088         struct ixl_queue        *que = vsi->queues;
3089         device_t                dev = vsi->dev;
3090         struct i40e_hw          *hw = vsi->hw;
3091         struct i40e_vsi_context ctxt;
3092         int                     err = 0;
3093
3094         memset(&ctxt, 0, sizeof(ctxt));
3095         ctxt.seid = vsi->seid;
3096         if (pf->veb_seid != 0)
3097                 ctxt.uplink_seid = pf->veb_seid;
3098         ctxt.pf_num = hw->pf_id;
3099         err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
3100         if (err) {
3101                 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d"
3102                     " aq_error %d\n", err, hw->aq.asq_last_status);
3103                 return (err);
3104         }
3105 #ifdef IXL_DEBUG
3106         device_printf(dev, "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
3107             "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
3108             "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
3109             ctxt.uplink_seid, ctxt.vsi_number,
3110             ctxt.vsis_allocated, ctxt.vsis_unallocated,
3111             ctxt.flags, ctxt.pf_num, ctxt.vf_num,
3112             ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
3113 #endif
3114         /*
3115         ** Set the queue and traffic class bits
3116         **  - when multiple traffic classes are supported
3117         **    this will need to be more robust.
3118         */
3119         ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
3120         ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
3121         /* In contig mode, que_mapping[0] is first queue index used by this VSI */
3122         ctxt.info.queue_mapping[0] = 0;
3123         /*
3124          * This VSI will only use traffic class 0; start traffic class 0's
3125          * queue allocation at queue 0, and assign it 64 (2^6) queues (though
3126          * the driver may not use all of them).
3127          */
3128         ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
3129             & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
3130             ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
3131             & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
3132
3133         /* Set VLAN receive stripping mode */
3134         ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
3135         ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
3136         if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
3137                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3138         else
3139                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3140
3141         /* Keep copy of VSI info in VSI for statistic counters */
3142         memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
3143
3144         /* Reset VSI statistics */
3145         ixl_vsi_reset_stats(vsi);
3146         vsi->hw_filters_add = 0;
3147         vsi->hw_filters_del = 0;
3148
3149         ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
3150
3151         err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3152         if (err) {
3153                 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d, aq_error %d\n",
3154                    err, hw->aq.asq_last_status);
3155                 return (err);
3156         }
3157
3158         for (int i = 0; i < vsi->num_queues; i++, que++) {
3159                 struct tx_ring          *txr = &que->txr;
3160                 struct rx_ring          *rxr = &que->rxr;
3161                 struct i40e_hmc_obj_txq tctx;
3162                 struct i40e_hmc_obj_rxq rctx;
3163                 u32                     txctl;
3164                 u16                     size;
3165
3166                 /* Setup the HMC TX Context  */
3167                 size = que->num_desc * sizeof(struct i40e_tx_desc);
3168                 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
3169                 tctx.new_context = 1;
3170                 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
3171                 tctx.qlen = que->num_desc;
3172                 tctx.fc_ena = 0;
3173                 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
3174                 /* Enable HEAD writeback */
3175                 tctx.head_wb_ena = 1;
3176                 tctx.head_wb_addr = txr->dma.pa +
3177                     (que->num_desc * sizeof(struct i40e_tx_desc));
3178                 tctx.rdylist_act = 0;
3179                 err = i40e_clear_lan_tx_queue_context(hw, i);
3180                 if (err) {
3181                         device_printf(dev, "Unable to clear TX context\n");
3182                         break;
3183                 }
3184                 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
3185                 if (err) {
3186                         device_printf(dev, "Unable to set TX context\n");
3187                         break;
3188                 }
3189                 /* Associate the ring with this PF */
3190                 txctl = I40E_QTX_CTL_PF_QUEUE;
3191                 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3192                     I40E_QTX_CTL_PF_INDX_MASK);
3193                 wr32(hw, I40E_QTX_CTL(i), txctl);
3194                 ixl_flush(hw);
3195
3196                 /* Do ring (re)init */
3197                 ixl_init_tx_ring(que);
3198
3199                 /* Next setup the HMC RX Context  */
3200                 if (vsi->max_frame_size <= MCLBYTES)
3201                         rxr->mbuf_sz = MCLBYTES;
3202                 else
3203                         rxr->mbuf_sz = MJUMPAGESIZE;
3204
3205                 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
3206
3207                 /* Set up an RX context for the HMC */
3208                 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
3209                 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
3210                 /* ignore header split for now */
3211                 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
3212                 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
3213                     vsi->max_frame_size : max_rxmax;
3214                 rctx.dtype = 0;
3215                 rctx.dsize = 1; /* do 32byte descriptors */
3216                 rctx.hsplit_0 = 0;  /* no HDR split initially */
3217                 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
3218                 rctx.qlen = que->num_desc;
3219                 rctx.tphrdesc_ena = 1;
3220                 rctx.tphwdesc_ena = 1;
3221                 rctx.tphdata_ena = 0;
3222                 rctx.tphhead_ena = 0;
3223                 rctx.lrxqthresh = 2;
3224                 rctx.crcstrip = 1;
3225                 rctx.l2tsel = 1;
3226                 rctx.showiv = 1;
3227                 rctx.fc_ena = 0;
3228                 rctx.prefena = 1;
3229
3230                 err = i40e_clear_lan_rx_queue_context(hw, i);
3231                 if (err) {
3232                         device_printf(dev,
3233                             "Unable to clear RX context %d\n", i);
3234                         break;
3235                 }
3236                 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
3237                 if (err) {
3238                         device_printf(dev, "Unable to set RX context %d\n", i);
3239                         break;
3240                 }
3241                 err = ixl_init_rx_ring(que);
3242                 if (err) {
3243                         device_printf(dev, "Fail in init_rx_ring %d\n", i);
3244                         break;
3245                 }
3246 #ifdef DEV_NETMAP
3247                 /* preserve queue */
3248                 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
3249                         struct netmap_adapter *na = NA(vsi->ifp);
3250                         struct netmap_kring *kring = &na->rx_rings[i];
3251                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
3252                         wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
3253                 } else
3254 #endif /* DEV_NETMAP */
3255                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
3256         }
3257         return (err);
3258 }
3259
3260
3261 /*********************************************************************
3262  *
3263  *  Free all VSI structs.
3264  *
3265  **********************************************************************/
3266 void
3267 ixl_free_vsi(struct ixl_vsi *vsi)
3268 {
3269         struct ixl_pf           *pf = (struct ixl_pf *)vsi->back;
3270         struct ixl_queue        *que = vsi->queues;
3271
3272         /* Free station queues */
3273         if (!vsi->queues)
3274                 goto free_filters;
3275
3276         for (int i = 0; i < vsi->num_queues; i++, que++) {
3277                 struct tx_ring *txr = &que->txr;
3278                 struct rx_ring *rxr = &que->rxr;
3279
3280                 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
3281                         continue;
3282                 IXL_TX_LOCK(txr);
3283                 ixl_free_que_tx(que);
3284                 if (txr->base)
3285                         i40e_free_dma_mem(&pf->hw, &txr->dma);
3286                 IXL_TX_UNLOCK(txr);
3287                 IXL_TX_LOCK_DESTROY(txr);
3288
3289                 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
3290                         continue;
3291                 IXL_RX_LOCK(rxr);
3292                 ixl_free_que_rx(que);
3293                 if (rxr->base)
3294                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
3295                 IXL_RX_UNLOCK(rxr);
3296                 IXL_RX_LOCK_DESTROY(rxr);
3297
3298         }
3299         free(vsi->queues, M_DEVBUF);
3300
3301 free_filters:
3302         /* Free VSI filter list */
3303         ixl_free_mac_filters(vsi);
3304 }
3305
3306 static void
3307 ixl_free_mac_filters(struct ixl_vsi *vsi)
3308 {
3309         struct ixl_mac_filter *f;
3310
3311         while (!SLIST_EMPTY(&vsi->ftl)) {
3312                 f = SLIST_FIRST(&vsi->ftl);
3313                 SLIST_REMOVE_HEAD(&vsi->ftl, next);
3314                 free(f, M_DEVBUF);
3315         }
3316 }
3317
3318
3319 /*********************************************************************
3320  *
3321  *  Allocate memory for the VSI (virtual station interface) and their
3322  *  associated queues, rings and the descriptors associated with each,
3323  *  called only once at attach.
3324  *
3325  **********************************************************************/
3326 static int
3327 ixl_setup_stations(struct ixl_pf *pf)
3328 {
3329         device_t                dev = pf->dev;
3330         struct ixl_vsi          *vsi;
3331         struct ixl_queue        *que;
3332         struct tx_ring          *txr;
3333         struct rx_ring          *rxr;
3334         int                     rsize, tsize;
3335         int                     error = I40E_SUCCESS;
3336
3337         vsi = &pf->vsi;
3338         vsi->back = (void *)pf;
3339         vsi->hw = &pf->hw;
3340         vsi->id = 0;
3341         vsi->num_vlans = 0;
3342         vsi->back = pf;
3343
3344         /* Get memory for the station queues */
3345         if (!(vsi->queues =
3346             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
3347             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3348                 device_printf(dev, "Unable to allocate queue memory\n");
3349                 error = ENOMEM;
3350                 goto early;
3351         }
3352
3353         for (int i = 0; i < vsi->num_queues; i++) {
3354                 que = &vsi->queues[i];
3355                 que->num_desc = ixl_ringsz;
3356                 que->me = i;
3357                 que->vsi = vsi;
3358                 /* mark the queue as active */
3359                 vsi->active_queues |= (u64)1 << que->me;
3360                 txr = &que->txr;
3361                 txr->que = que;
3362                 txr->tail = I40E_QTX_TAIL(que->me);
3363
3364                 /* Initialize the TX lock */
3365                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3366                     device_get_nameunit(dev), que->me);
3367                 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
3368                 /* Create the TX descriptor ring */
3369                 tsize = roundup2((que->num_desc *
3370                     sizeof(struct i40e_tx_desc)) +
3371                     sizeof(u32), DBA_ALIGN);
3372                 if (i40e_allocate_dma_mem(&pf->hw,
3373                     &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
3374                         device_printf(dev,
3375                             "Unable to allocate TX Descriptor memory\n");
3376                         error = ENOMEM;
3377                         goto fail;
3378                 }
3379                 txr->base = (struct i40e_tx_desc *)txr->dma.va;
3380                 bzero((void *)txr->base, tsize);
3381                 /* Now allocate transmit soft structs for the ring */
3382                 if (ixl_allocate_tx_data(que)) {
3383                         device_printf(dev,
3384                             "Critical Failure setting up TX structures\n");
3385                         error = ENOMEM;
3386                         goto fail;
3387                 }
3388                 /* Allocate a buf ring */
3389                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3390                     M_NOWAIT, &txr->mtx);
3391                 if (txr->br == NULL) {
3392                         device_printf(dev,
3393                             "Critical Failure setting up TX buf ring\n");
3394                         error = ENOMEM;
3395                         goto fail;
3396                 }
3397
3398                 /*
3399                  * Next the RX queues...
3400                  */
3401                 rsize = roundup2(que->num_desc *
3402                     sizeof(union i40e_rx_desc), DBA_ALIGN);
3403                 rxr = &que->rxr;
3404                 rxr->que = que;
3405                 rxr->tail = I40E_QRX_TAIL(que->me);
3406
3407                 /* Initialize the RX side lock */
3408                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3409                     device_get_nameunit(dev), que->me);
3410                 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3411
3412                 if (i40e_allocate_dma_mem(&pf->hw,
3413                     &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3414                         device_printf(dev,
3415                             "Unable to allocate RX Descriptor memory\n");
3416                         error = ENOMEM;
3417                         goto fail;
3418                 }
3419                 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3420                 bzero((void *)rxr->base, rsize);
3421
3422                 /* Allocate receive soft structs for the ring*/
3423                 if (ixl_allocate_rx_data(que)) {
3424                         device_printf(dev,
3425                             "Critical Failure setting up receive structs\n");
3426                         error = ENOMEM;
3427                         goto fail;
3428                 }
3429         }
3430
3431         return (0);
3432
3433 fail:
3434         for (int i = 0; i < vsi->num_queues; i++) {
3435                 que = &vsi->queues[i];
3436                 rxr = &que->rxr;
3437                 txr = &que->txr;
3438                 if (rxr->base)
3439                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
3440                 if (txr->base)
3441                         i40e_free_dma_mem(&pf->hw, &txr->dma);
3442         }
3443
3444 early:
3445         return (error);
3446 }
3447
3448 /*
3449 ** Provide a update to the queue RX
3450 ** interrupt moderation value.
3451 */
3452 static void
3453 ixl_set_queue_rx_itr(struct ixl_queue *que)
3454 {
3455         struct ixl_vsi  *vsi = que->vsi;
3456         struct i40e_hw  *hw = vsi->hw;
3457         struct rx_ring  *rxr = &que->rxr;
3458         u16             rx_itr;
3459         u16             rx_latency = 0;
3460         int             rx_bytes;
3461
3462         /* Idle, do nothing */
3463         if (rxr->bytes == 0)
3464                 return;
3465
3466         if (ixl_dynamic_rx_itr) {
3467                 rx_bytes = rxr->bytes/rxr->itr;
3468                 rx_itr = rxr->itr;
3469
3470                 /* Adjust latency range */
3471                 switch (rxr->latency) {
3472                 case IXL_LOW_LATENCY:
3473                         if (rx_bytes > 10) {
3474                                 rx_latency = IXL_AVE_LATENCY;
3475                                 rx_itr = IXL_ITR_20K;
3476                         }
3477                         break;
3478                 case IXL_AVE_LATENCY:
3479                         if (rx_bytes > 20) {
3480                                 rx_latency = IXL_BULK_LATENCY;
3481                                 rx_itr = IXL_ITR_8K;
3482                         } else if (rx_bytes <= 10) {
3483                                 rx_latency = IXL_LOW_LATENCY;
3484                                 rx_itr = IXL_ITR_100K;
3485                         }
3486                         break;
3487                 case IXL_BULK_LATENCY:
3488                         if (rx_bytes <= 20) {
3489                                 rx_latency = IXL_AVE_LATENCY;
3490                                 rx_itr = IXL_ITR_20K;
3491                         }
3492                         break;
3493                  }
3494
3495                 rxr->latency = rx_latency;
3496
3497                 if (rx_itr != rxr->itr) {
3498                         /* do an exponential smoothing */
3499                         rx_itr = (10 * rx_itr * rxr->itr) /
3500                             ((9 * rx_itr) + rxr->itr);
3501                         rxr->itr = rx_itr & IXL_MAX_ITR;
3502                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3503                             que->me), rxr->itr);
3504                 }
3505         } else { /* We may have have toggled to non-dynamic */
3506                 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3507                         vsi->rx_itr_setting = ixl_rx_itr;
3508                 /* Update the hardware if needed */
3509                 if (rxr->itr != vsi->rx_itr_setting) {
3510                         rxr->itr = vsi->rx_itr_setting;
3511                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3512                             que->me), rxr->itr);
3513                 }
3514         }
3515         rxr->bytes = 0;
3516         rxr->packets = 0;
3517         return;
3518 }
3519
3520
3521 /*
3522 ** Provide a update to the queue TX
3523 ** interrupt moderation value.
3524 */
3525 static void
3526 ixl_set_queue_tx_itr(struct ixl_queue *que)
3527 {
3528         struct ixl_vsi  *vsi = que->vsi;
3529         struct i40e_hw  *hw = vsi->hw;
3530         struct tx_ring  *txr = &que->txr;
3531         u16             tx_itr;
3532         u16             tx_latency = 0;
3533         int             tx_bytes;
3534
3535
3536         /* Idle, do nothing */
3537         if (txr->bytes == 0)
3538                 return;
3539
3540         if (ixl_dynamic_tx_itr) {
3541                 tx_bytes = txr->bytes/txr->itr;
3542                 tx_itr = txr->itr;
3543
3544                 switch (txr->latency) {
3545                 case IXL_LOW_LATENCY:
3546                         if (tx_bytes > 10) {
3547                                 tx_latency = IXL_AVE_LATENCY;
3548                                 tx_itr = IXL_ITR_20K;
3549                         }
3550                         break;
3551                 case IXL_AVE_LATENCY:
3552                         if (tx_bytes > 20) {
3553                                 tx_latency = IXL_BULK_LATENCY;
3554                                 tx_itr = IXL_ITR_8K;
3555                         } else if (tx_bytes <= 10) {
3556                                 tx_latency = IXL_LOW_LATENCY;
3557                                 tx_itr = IXL_ITR_100K;
3558                         }
3559                         break;
3560                 case IXL_BULK_LATENCY:
3561                         if (tx_bytes <= 20) {
3562                                 tx_latency = IXL_AVE_LATENCY;
3563                                 tx_itr = IXL_ITR_20K;
3564                         }
3565                         break;
3566                 }
3567
3568                 txr->latency = tx_latency;
3569
3570                 if (tx_itr != txr->itr) {
3571                  /* do an exponential smoothing */
3572                         tx_itr = (10 * tx_itr * txr->itr) /
3573                             ((9 * tx_itr) + txr->itr);
3574                         txr->itr = tx_itr & IXL_MAX_ITR;
3575                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3576                             que->me), txr->itr);
3577                 }
3578
3579         } else { /* We may have have toggled to non-dynamic */
3580                 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3581                         vsi->tx_itr_setting = ixl_tx_itr;
3582                 /* Update the hardware if needed */
3583                 if (txr->itr != vsi->tx_itr_setting) {
3584                         txr->itr = vsi->tx_itr_setting;
3585                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3586                             que->me), txr->itr);
3587                 }
3588         }
3589         txr->bytes = 0;
3590         txr->packets = 0;
3591         return;
3592 }
3593
3594 #define QUEUE_NAME_LEN 32
3595
3596 static void
3597 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3598     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3599 {
3600         struct sysctl_oid *tree;
3601         struct sysctl_oid_list *child;
3602         struct sysctl_oid_list *vsi_list;
3603
3604         tree = device_get_sysctl_tree(pf->dev);
3605         child = SYSCTL_CHILDREN(tree);
3606         vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3607                                    CTLFLAG_RD, NULL, "VSI Number");
3608         vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3609
3610         ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3611 }
3612
3613 #ifdef IXL_DEBUG
3614 /**
3615  * ixl_sysctl_qtx_tail_handler
3616  * Retrieves I40E_QTX_TAIL value from hardware
3617  * for a sysctl.
3618  */
3619 static int
3620 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)
3621 {
3622         struct ixl_queue *que;
3623         int error;
3624         u32 val;
3625
3626         que = ((struct ixl_queue *)oidp->oid_arg1);
3627         if (!que) return 0;
3628
3629         val = rd32(que->vsi->hw, que->txr.tail);
3630         error = sysctl_handle_int(oidp, &val, 0, req);
3631         if (error || !req->newptr)
3632                 return error;
3633         return (0);
3634 }
3635
3636 /**
3637  * ixl_sysctl_qrx_tail_handler
3638  * Retrieves I40E_QRX_TAIL value from hardware
3639  * for a sysctl.
3640  */
3641 static int
3642 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)
3643 {
3644         struct ixl_queue *que;
3645         int error;
3646         u32 val;
3647
3648         que = ((struct ixl_queue *)oidp->oid_arg1);
3649         if (!que) return 0;
3650
3651         val = rd32(que->vsi->hw, que->rxr.tail);
3652         error = sysctl_handle_int(oidp, &val, 0, req);
3653         if (error || !req->newptr)
3654                 return error;
3655         return (0);
3656 }
3657 #endif
3658
3659 static void
3660 ixl_add_hw_stats(struct ixl_pf *pf)
3661 {
3662         device_t dev = pf->dev;
3663         struct ixl_vsi *vsi = &pf->vsi;
3664         struct ixl_queue *queues = vsi->queues;
3665         struct i40e_hw_port_stats *pf_stats = &pf->stats;
3666
3667         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3668         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3669         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3670         struct sysctl_oid_list *vsi_list;
3671
3672         struct sysctl_oid *queue_node;
3673         struct sysctl_oid_list *queue_list;
3674
3675         struct tx_ring *txr;
3676         struct rx_ring *rxr;
3677         char queue_namebuf[QUEUE_NAME_LEN];
3678
3679         /* Driver statistics */
3680         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3681                         CTLFLAG_RD, &pf->watchdog_events,
3682                         "Watchdog timeouts");
3683         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3684                         CTLFLAG_RD, &pf->admin_irq,
3685                         "Admin Queue IRQ Handled");
3686
3687         ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3688         vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3689
3690         /* Queue statistics */
3691         for (int q = 0; q < vsi->num_queues; q++) {
3692                 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3693                 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3694                     OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3695                 queue_list = SYSCTL_CHILDREN(queue_node);
3696
3697                 txr = &(queues[q].txr);
3698                 rxr = &(queues[q].rxr);
3699
3700                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3701                                 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3702                                 "m_defrag() failed");
3703                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3704                                 CTLFLAG_RD, &(queues[q].irqs),
3705                                 "irqs on this queue");
3706                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3707                                 CTLFLAG_RD, &(queues[q].tso),
3708                                 "TSO");
3709                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3710                                 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3711                                 "Driver tx dma failure in xmit");
3712                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3713                                 CTLFLAG_RD, &(txr->no_desc),
3714                                 "Queue No Descriptor Available");
3715                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3716                                 CTLFLAG_RD, &(txr->total_packets),
3717                                 "Queue Packets Transmitted");
3718                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3719                                 CTLFLAG_RD, &(txr->tx_bytes),
3720                                 "Queue Bytes Transmitted");
3721                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3722                                 CTLFLAG_RD, &(rxr->rx_packets),
3723                                 "Queue Packets Received");
3724                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3725                                 CTLFLAG_RD, &(rxr->rx_bytes),
3726                                 "Queue Bytes Received");
3727                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err",
3728                                 CTLFLAG_RD, &(rxr->desc_errs),
3729                                 "Queue Rx Descriptor Errors");
3730                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr",
3731                                 CTLFLAG_RD, &(rxr->itr), 0,
3732                                 "Queue Rx ITR Interval");
3733                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr",
3734                                 CTLFLAG_RD, &(txr->itr), 0,
3735                                 "Queue Tx ITR Interval");
3736                 // Not actual latency; just a calculated value to put in a register
3737                 // TODO: Put in better descriptions here
3738                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_latency",
3739                                 CTLFLAG_RD, &(rxr->latency), 0,
3740                                 "Queue Rx ITRL Average Interval");
3741                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_latency",
3742                                 CTLFLAG_RD, &(txr->latency), 0,
3743                                 "Queue Tx ITRL Average Interval");
3744
3745 #ifdef IXL_DEBUG
3746                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done",
3747                                 CTLFLAG_RD, &(rxr->not_done),
3748                                 "Queue Rx Descriptors not Done");
3749                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh",
3750                                 CTLFLAG_RD, &(rxr->next_refresh), 0,
3751                                 "Queue Rx Descriptors not Done");
3752                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check",
3753                                 CTLFLAG_RD, &(rxr->next_check), 0,
3754                                 "Queue Rx Descriptors not Done");
3755                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail",
3756                                 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3757                                 sizeof(struct ixl_queue),
3758                                 ixl_sysctl_qtx_tail_handler, "IU",
3759                                 "Queue Transmit Descriptor Tail");
3760                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail",
3761                                 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3762                                 sizeof(struct ixl_queue),
3763                                 ixl_sysctl_qrx_tail_handler, "IU",
3764                                 "Queue Receive Descriptor Tail");
3765 #endif
3766         }
3767
3768         /* MAC stats */
3769         ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3770 }
3771
3772 static void
3773 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3774         struct sysctl_oid_list *child,
3775         struct i40e_eth_stats *eth_stats)
3776 {
3777         struct ixl_sysctl_info ctls[] =
3778         {
3779                 {&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3780                 {&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3781                         "Unicast Packets Received"},
3782                 {&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3783                         "Multicast Packets Received"},
3784                 {&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3785                         "Broadcast Packets Received"},
3786                 {&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3787                 {&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3788                 {&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3789                 {&eth_stats->tx_multicast, "mcast_pkts_txd",
3790                         "Multicast Packets Transmitted"},
3791                 {&eth_stats->tx_broadcast, "bcast_pkts_txd",
3792                         "Broadcast Packets Transmitted"},
3793                 // end
3794                 {0,0,0}
3795         };
3796
3797         struct ixl_sysctl_info *entry = ctls;
3798         while (entry->stat != NULL)
3799         {
3800                 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3801                                 CTLFLAG_RD, entry->stat,
3802                                 entry->description);
3803                 entry++;
3804         }
3805 }
3806
3807 static void
3808 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3809         struct sysctl_oid_list *child,
3810         struct i40e_hw_port_stats *stats)
3811 {
3812         struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3813                                     CTLFLAG_RD, NULL, "Mac Statistics");
3814         struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3815
3816         struct i40e_eth_stats *eth_stats = &stats->eth;
3817         ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3818
3819         struct ixl_sysctl_info ctls[] =
3820         {
3821                 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3822                 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3823                 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3824                 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3825                 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3826                 /* Packet Reception Stats */
3827                 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3828                 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3829                 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3830                 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3831                 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3832                 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3833                 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3834                 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3835                 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3836                 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3837                 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3838                 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3839                 /* Packet Transmission Stats */
3840                 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3841                 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3842                 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3843                 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3844                 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3845                 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3846                 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3847                 /* Flow control */
3848                 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3849                 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3850                 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3851                 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3852                 /* End */
3853                 {0,0,0}
3854         };
3855
3856         struct ixl_sysctl_info *entry = ctls;
3857         while (entry->stat != NULL)
3858         {
3859                 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3860                                 CTLFLAG_RD, entry->stat,
3861                                 entry->description);
3862                 entry++;
3863         }
3864 }
3865
3866
3867 /*
3868 ** ixl_config_rss - setup RSS
3869 **  - note this is done for the single vsi
3870 */
3871 static void
3872 ixl_config_rss(struct ixl_vsi *vsi)
3873 {
3874         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3875         struct i40e_hw  *hw = vsi->hw;
3876         u32             lut = 0;
3877         u64             set_hena = 0, hena;
3878         int             i, j, que_id;
3879 #ifdef RSS
3880         u32             rss_hash_config;
3881         u32             rss_seed[IXL_KEYSZ];
3882 #else
3883         u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3884                             0x183cfd8c, 0xce880440, 0x580cbc3c,
3885                             0x35897377, 0x328b25e1, 0x4fa98922,
3886                             0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3887 #endif
3888
3889 #ifdef RSS
3890         /* Fetch the configured RSS key */
3891         rss_getkey((uint8_t *) &rss_seed);
3892 #endif
3893
3894         /* Fill out hash function seed */
3895         for (i = 0; i < IXL_KEYSZ; i++)
3896                 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3897
3898         /* Enable PCTYPES for RSS: */
3899 #ifdef RSS
3900         rss_hash_config = rss_gethashconfig();
3901         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3902                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3903         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3904                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3905         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3906                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3907         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3908                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3909         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3910                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3911         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3912                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3913         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3914                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3915 #else
3916         set_hena =
3917                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3918                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3919                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3920                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3921                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3922                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3923                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3924                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3925                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3926                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3927                 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3928 #endif
3929         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
3930             ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
3931         hena |= set_hena;
3932         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
3933         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3934
3935         /* Populate the LUT with max no. of queues in round robin fashion */
3936         for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3937                 if (j == vsi->num_queues)
3938                         j = 0;
3939 #ifdef RSS
3940                 /*
3941                  * Fetch the RSS bucket id for the given indirection entry.
3942                  * Cap it at the number of configured buckets (which is
3943                  * num_queues.)
3944                  */
3945                 que_id = rss_get_indirection_to_bucket(i);
3946                 que_id = que_id % vsi->num_queues;
3947 #else
3948                 que_id = j;
3949 #endif
3950                 /* lut = 4-byte sliding window of 4 lut entries */
3951                 lut = (lut << 8) | (que_id &
3952                     ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3953                 /* On i = 3, we have 4 entries in lut; write to the register */
3954                 if ((i & 3) == 3)
3955                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3956         }
3957         ixl_flush(hw);
3958 }
3959
3960
3961 /*
3962 ** This routine is run via an vlan config EVENT,
3963 ** it enables us to use the HW Filter table since
3964 ** we can get the vlan id. This just creates the
3965 ** entry in the soft version of the VFTA, init will
3966 ** repopulate the real table.
3967 */
3968 static void
3969 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3970 {
3971         struct ixl_vsi  *vsi = ifp->if_softc;
3972         struct i40e_hw  *hw = vsi->hw;
3973         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3974
3975         if (ifp->if_softc !=  arg)   /* Not our event */
3976                 return;
3977
3978         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3979                 return;
3980
3981         IXL_PF_LOCK(pf);
3982         ++vsi->num_vlans;
3983         ixl_add_filter(vsi, hw->mac.addr, vtag);
3984         IXL_PF_UNLOCK(pf);
3985 }
3986
3987 /*
3988 ** This routine is run via an vlan
3989 ** unconfig EVENT, remove our entry
3990 ** in the soft vfta.
3991 */
3992 static void
3993 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3994 {
3995         struct ixl_vsi  *vsi = ifp->if_softc;
3996         struct i40e_hw  *hw = vsi->hw;
3997         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3998
3999         if (ifp->if_softc !=  arg)
4000                 return;
4001
4002         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4003                 return;
4004
4005         IXL_PF_LOCK(pf);
4006         --vsi->num_vlans;
4007         ixl_del_filter(vsi, hw->mac.addr, vtag);
4008         IXL_PF_UNLOCK(pf);
4009 }
4010
4011 /*
4012 ** This routine updates vlan filters, called by init
4013 ** it scans the filter table and then updates the hw
4014 ** after a soft reset.
4015 */
4016 static void
4017 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
4018 {
4019         struct ixl_mac_filter   *f;
4020         int                     cnt = 0, flags;
4021
4022         if (vsi->num_vlans == 0)
4023                 return;
4024         /*
4025         ** Scan the filter list for vlan entries,
4026         ** mark them for addition and then call
4027         ** for the AQ update.
4028         */
4029         SLIST_FOREACH(f, &vsi->ftl, next) {
4030                 if (f->flags & IXL_FILTER_VLAN) {
4031                         f->flags |=
4032                             (IXL_FILTER_ADD |
4033                             IXL_FILTER_USED);
4034                         cnt++;
4035                 }
4036         }
4037         if (cnt == 0) {
4038                 printf("setup vlan: no filters found!\n");
4039                 return;
4040         }
4041         flags = IXL_FILTER_VLAN;
4042         flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4043         ixl_add_hw_filters(vsi, flags, cnt);
4044         return;
4045 }
4046
4047 /*
4048 ** Initialize filter list and add filters that the hardware
4049 ** needs to know about.
4050 **
4051 ** Requires VSI's filter list & seid to be set before calling.
4052 */
4053 static void
4054 ixl_init_filters(struct ixl_vsi *vsi)
4055 {
4056         /* Add broadcast address */
4057         ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
4058
4059         /*
4060          * Prevent Tx flow control frames from being sent out by
4061          * non-firmware transmitters.
4062          */
4063         i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid);
4064 }
4065
4066 /*
4067 ** This routine adds mulicast filters
4068 */
4069 static void
4070 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
4071 {
4072         struct ixl_mac_filter *f;
4073
4074         /* Does one already exist */
4075         f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4076         if (f != NULL)
4077                 return;
4078
4079         f = ixl_get_filter(vsi);
4080         if (f == NULL) {
4081                 printf("WARNING: no filter available!!\n");
4082                 return;
4083         }
4084         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4085         f->vlan = IXL_VLAN_ANY;
4086         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
4087             | IXL_FILTER_MC);
4088
4089         return;
4090 }
4091
4092 static void
4093 ixl_reconfigure_filters(struct ixl_vsi *vsi)
4094 {
4095
4096         ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
4097 }
4098
4099 /*
4100 ** This routine adds macvlan filters
4101 */
4102 static void
4103 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4104 {
4105         struct ixl_mac_filter   *f, *tmp;
4106         struct ixl_pf           *pf;
4107         device_t                dev;
4108
4109         DEBUGOUT("ixl_add_filter: begin");
4110
4111         pf = vsi->back;
4112         dev = pf->dev;
4113
4114         /* Does one already exist */
4115         f = ixl_find_filter(vsi, macaddr, vlan);
4116         if (f != NULL)
4117                 return;
4118         /*
4119         ** Is this the first vlan being registered, if so we
4120         ** need to remove the ANY filter that indicates we are
4121         ** not in a vlan, and replace that with a 0 filter.
4122         */
4123         if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
4124                 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4125                 if (tmp != NULL) {
4126                         ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
4127                         ixl_add_filter(vsi, macaddr, 0);
4128                 }
4129         }
4130
4131         f = ixl_get_filter(vsi);
4132         if (f == NULL) {
4133                 device_printf(dev, "WARNING: no filter available!!\n");
4134                 return;
4135         }
4136         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4137         f->vlan = vlan;
4138         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4139         if (f->vlan != IXL_VLAN_ANY)
4140                 f->flags |= IXL_FILTER_VLAN;
4141         else
4142                 vsi->num_macs++;
4143
4144         ixl_add_hw_filters(vsi, f->flags, 1);
4145         return;
4146 }
4147
4148 static void
4149 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4150 {
4151         struct ixl_mac_filter *f;
4152
4153         f = ixl_find_filter(vsi, macaddr, vlan);
4154         if (f == NULL)
4155                 return;
4156
4157         f->flags |= IXL_FILTER_DEL;
4158         ixl_del_hw_filters(vsi, 1);
4159         vsi->num_macs--;
4160
4161         /* Check if this is the last vlan removal */
4162         if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
4163                 /* Switch back to a non-vlan filter */
4164                 ixl_del_filter(vsi, macaddr, 0);
4165                 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
4166         }
4167         return;
4168 }
4169
4170 /*
4171 ** Find the filter with both matching mac addr and vlan id
4172 */
4173 static struct ixl_mac_filter *
4174 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4175 {
4176         struct ixl_mac_filter   *f;
4177         bool                    match = FALSE;
4178
4179         SLIST_FOREACH(f, &vsi->ftl, next) {
4180                 if (!cmp_etheraddr(f->macaddr, macaddr))
4181                         continue;
4182                 if (f->vlan == vlan) {
4183                         match = TRUE;
4184                         break;
4185                 }
4186         }
4187
4188         if (!match)
4189                 f = NULL;
4190         return (f);
4191 }
4192
4193 /*
4194 ** This routine takes additions to the vsi filter
4195 ** table and creates an Admin Queue call to create
4196 ** the filters in the hardware.
4197 */
4198 static void
4199 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
4200 {
4201         struct i40e_aqc_add_macvlan_element_data *a, *b;
4202         struct ixl_mac_filter   *f;
4203         struct ixl_pf           *pf;
4204         struct i40e_hw          *hw;
4205         device_t                dev;
4206         int                     err, j = 0;
4207
4208         pf = vsi->back;
4209         dev = pf->dev;
4210         hw = &pf->hw;
4211         IXL_PF_LOCK_ASSERT(pf);
4212
4213         a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
4214             M_DEVBUF, M_NOWAIT | M_ZERO);
4215         if (a == NULL) {
4216                 device_printf(dev, "add_hw_filters failed to get memory\n");
4217                 return;
4218         }
4219
4220         /*
4221         ** Scan the filter list, each time we find one
4222         ** we add it to the admin queue array and turn off
4223         ** the add bit.
4224         */
4225         SLIST_FOREACH(f, &vsi->ftl, next) {
4226                 if (f->flags == flags) {
4227                         b = &a[j]; // a pox on fvl long names :)
4228                         bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
4229                         if (f->vlan == IXL_VLAN_ANY) {
4230                                 b->vlan_tag = 0;
4231                                 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
4232                         } else {
4233                                 b->vlan_tag = f->vlan;
4234                                 b->flags = 0;
4235                         }
4236                         b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
4237                         f->flags &= ~IXL_FILTER_ADD;
4238                         j++;
4239                 }
4240                 if (j == cnt)
4241                         break;
4242         }
4243         if (j > 0) {
4244                 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
4245                 if (err)
4246                         device_printf(dev, "aq_add_macvlan err %d, "
4247                             "aq_error %d\n", err, hw->aq.asq_last_status);
4248                 else
4249                         vsi->hw_filters_add += j;
4250         }
4251         free(a, M_DEVBUF);
4252         return;
4253 }
4254
4255 /*
4256 ** This routine takes removals in the vsi filter
4257 ** table and creates an Admin Queue call to delete
4258 ** the filters in the hardware.
4259 */
4260 static void
4261 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
4262 {
4263         struct i40e_aqc_remove_macvlan_element_data *d, *e;
4264         struct ixl_pf           *pf;
4265         struct i40e_hw          *hw;
4266         device_t                dev;
4267         struct ixl_mac_filter   *f, *f_temp;
4268         int                     err, j = 0;
4269
4270         DEBUGOUT("ixl_del_hw_filters: begin\n");
4271
4272         pf = vsi->back;
4273         hw = &pf->hw;
4274         dev = pf->dev;
4275
4276         d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
4277             M_DEVBUF, M_NOWAIT | M_ZERO);
4278         if (d == NULL) {
4279                 printf("del hw filter failed to get memory\n");
4280                 return;
4281         }
4282
4283         SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
4284                 if (f->flags & IXL_FILTER_DEL) {
4285                         e = &d[j]; // a pox on fvl long names :)
4286                         bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
4287                         e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
4288                         e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
4289                         /* delete entry from vsi list */
4290                         SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
4291                         free(f, M_DEVBUF);
4292                         j++;
4293                 }
4294                 if (j == cnt)
4295                         break;
4296         }
4297         if (j > 0) {
4298                 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
4299                 /* NOTE: returns ENOENT every time but seems to work fine,
4300                    so we'll ignore that specific error. */
4301                 // TODO: Does this still occur on current firmwares?
4302                 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
4303                         int sc = 0;
4304                         for (int i = 0; i < j; i++)
4305                                 sc += (!d[i].error_code);
4306                         vsi->hw_filters_del += sc;
4307                         device_printf(dev,
4308                             "Failed to remove %d/%d filters, aq error %d\n",
4309                             j - sc, j, hw->aq.asq_last_status);
4310                 } else
4311                         vsi->hw_filters_del += j;
4312         }
4313         free(d, M_DEVBUF);
4314
4315         DEBUGOUT("ixl_del_hw_filters: end\n");
4316         return;
4317 }
4318
4319 static int
4320 ixl_enable_rings(struct ixl_vsi *vsi)
4321 {
4322         struct ixl_pf   *pf = vsi->back;
4323         struct i40e_hw  *hw = &pf->hw;
4324         int             index, error;
4325         u32             reg;
4326
4327         error = 0;
4328         for (int i = 0; i < vsi->num_queues; i++) {
4329                 index = vsi->first_queue + i;
4330                 i40e_pre_tx_queue_cfg(hw, index, TRUE);
4331
4332                 reg = rd32(hw, I40E_QTX_ENA(index));
4333                 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
4334                     I40E_QTX_ENA_QENA_STAT_MASK;
4335                 wr32(hw, I40E_QTX_ENA(index), reg);
4336                 /* Verify the enable took */
4337                 for (int j = 0; j < 10; j++) {
4338                         reg = rd32(hw, I40E_QTX_ENA(index));
4339                         if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4340                                 break;
4341                         i40e_msec_delay(10);
4342                 }
4343                 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
4344                         device_printf(pf->dev, "TX queue %d disabled!\n",
4345                             index);
4346                         error = ETIMEDOUT;
4347                 }
4348
4349                 reg = rd32(hw, I40E_QRX_ENA(index));
4350                 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
4351                     I40E_QRX_ENA_QENA_STAT_MASK;
4352                 wr32(hw, I40E_QRX_ENA(index), reg);
4353                 /* Verify the enable took */
4354                 for (int j = 0; j < 10; j++) {
4355                         reg = rd32(hw, I40E_QRX_ENA(index));
4356                         if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
4357                                 break;
4358                         i40e_msec_delay(10);
4359                 }
4360                 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
4361                         device_printf(pf->dev, "RX queue %d disabled!\n",
4362                             index);
4363                         error = ETIMEDOUT;
4364                 }
4365         }
4366
4367         return (error);
4368 }
4369
4370 static int
4371 ixl_disable_rings(struct ixl_vsi *vsi)
4372 {
4373         struct ixl_pf   *pf = vsi->back;
4374         struct i40e_hw  *hw = &pf->hw;
4375         int             index, error;
4376         u32             reg;
4377
4378         error = 0;
4379         for (int i = 0; i < vsi->num_queues; i++) {
4380                 index = vsi->first_queue + i;
4381
4382                 i40e_pre_tx_queue_cfg(hw, index, FALSE);
4383                 i40e_usec_delay(500);
4384
4385                 reg = rd32(hw, I40E_QTX_ENA(index));
4386                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4387                 wr32(hw, I40E_QTX_ENA(index), reg);
4388                 /* Verify the disable took */
4389                 for (int j = 0; j < 10; j++) {
4390                         reg = rd32(hw, I40E_QTX_ENA(index));
4391                         if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4392                                 break;
4393                         i40e_msec_delay(10);
4394                 }
4395                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
4396                         device_printf(pf->dev, "TX queue %d still enabled!\n",
4397                             index);
4398                         error = ETIMEDOUT;
4399                 }
4400
4401                 reg = rd32(hw, I40E_QRX_ENA(index));
4402                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4403                 wr32(hw, I40E_QRX_ENA(index), reg);
4404                 /* Verify the disable took */
4405                 for (int j = 0; j < 10; j++) {
4406                         reg = rd32(hw, I40E_QRX_ENA(index));
4407                         if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
4408                                 break;
4409                         i40e_msec_delay(10);
4410                 }
4411                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
4412                         device_printf(pf->dev, "RX queue %d still enabled!\n",
4413                             index);
4414                         error = ETIMEDOUT;
4415                 }
4416         }
4417
4418         return (error);
4419 }
4420
4421 /**
4422  * ixl_handle_mdd_event
4423  *
4424  * Called from interrupt handler to identify possibly malicious vfs
4425  * (But also detects events from the PF, as well)
4426  **/
4427 static void
4428 ixl_handle_mdd_event(struct ixl_pf *pf)
4429 {
4430         struct i40e_hw *hw = &pf->hw;
4431         device_t dev = pf->dev;
4432         bool mdd_detected = false;
4433         bool pf_mdd_detected = false;
4434         u32 reg;
4435
4436         /* find what triggered the MDD event */
4437         reg = rd32(hw, I40E_GL_MDET_TX);
4438         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
4439                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
4440                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
4441                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
4442                                 I40E_GL_MDET_TX_EVENT_SHIFT;
4443                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
4444                                 I40E_GL_MDET_TX_QUEUE_SHIFT;
4445                 device_printf(dev,
4446                          "Malicious Driver Detection event 0x%02x"
4447                          " on TX queue %d pf number 0x%02x\n",
4448                          event, queue, pf_num);
4449                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
4450                 mdd_detected = true;
4451         }
4452         reg = rd32(hw, I40E_GL_MDET_RX);
4453         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4454                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
4455                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
4456                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
4457                                 I40E_GL_MDET_RX_EVENT_SHIFT;
4458                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
4459                                 I40E_GL_MDET_RX_QUEUE_SHIFT;
4460                 device_printf(dev,
4461                          "Malicious Driver Detection event 0x%02x"
4462                          " on RX queue %d of function 0x%02x\n",
4463                          event, queue, func);
4464                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4465                 mdd_detected = true;
4466         }
4467
4468         if (mdd_detected) {
4469                 reg = rd32(hw, I40E_PF_MDET_TX);
4470                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4471                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4472                         device_printf(dev,
4473                                  "MDD TX event is for this function 0x%08x",
4474                                  reg);
4475                         pf_mdd_detected = true;
4476                 }
4477                 reg = rd32(hw, I40E_PF_MDET_RX);
4478                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4479                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4480                         device_printf(dev,
4481                                  "MDD RX event is for this function 0x%08x",
4482                                  reg);
4483                         pf_mdd_detected = true;
4484                 }
4485         }
4486
4487         /* re-enable mdd interrupt cause */
4488         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4489         reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4490         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4491         ixl_flush(hw);
4492 }
4493
4494 static void
4495 ixl_enable_intr(struct ixl_vsi *vsi)
4496 {
4497         struct i40e_hw          *hw = vsi->hw;
4498         struct ixl_queue        *que = vsi->queues;
4499
4500         if (ixl_enable_msix) {
4501                 for (int i = 0; i < vsi->num_queues; i++, que++)
4502                         ixl_enable_queue(hw, que->me);
4503         } else
4504                 ixl_enable_legacy(hw);
4505 }
4506
4507 static void
4508 ixl_disable_rings_intr(struct ixl_vsi *vsi)
4509 {
4510         struct i40e_hw          *hw = vsi->hw;
4511         struct ixl_queue        *que = vsi->queues;
4512
4513         for (int i = 0; i < vsi->num_queues; i++, que++)
4514                 ixl_disable_queue(hw, que->me);
4515 }
4516
4517 static void
4518 ixl_disable_intr(struct ixl_vsi *vsi)
4519 {
4520         struct i40e_hw          *hw = vsi->hw;
4521
4522         if (ixl_enable_msix)
4523                 ixl_disable_adminq(hw);
4524         else
4525                 ixl_disable_legacy(hw);
4526 }
4527
4528 static void
4529 ixl_enable_adminq(struct i40e_hw *hw)
4530 {
4531         u32             reg;
4532
4533         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4534             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4535             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4536         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4537         ixl_flush(hw);
4538 }
4539
4540 static void
4541 ixl_disable_adminq(struct i40e_hw *hw)
4542 {
4543         u32             reg;
4544
4545         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4546         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4547         ixl_flush(hw);
4548 }
4549
4550 static void
4551 ixl_enable_queue(struct i40e_hw *hw, int id)
4552 {
4553         u32             reg;
4554
4555         reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4556             I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4557             (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4558         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4559 }
4560
4561 static void
4562 ixl_disable_queue(struct i40e_hw *hw, int id)
4563 {
4564         u32             reg;
4565
4566         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4567         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4568 }
4569
4570 static void
4571 ixl_enable_legacy(struct i40e_hw *hw)
4572 {
4573         u32             reg;
4574         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4575             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4576             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4577         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4578 }
4579
4580 static void
4581 ixl_disable_legacy(struct i40e_hw *hw)
4582 {
4583         u32             reg;
4584
4585         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4586         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4587 }
4588
4589 static void
4590 ixl_update_stats_counters(struct ixl_pf *pf)
4591 {
4592         struct i40e_hw  *hw = &pf->hw;
4593         struct ixl_vsi  *vsi = &pf->vsi;
4594         struct ixl_vf   *vf;
4595
4596         struct i40e_hw_port_stats *nsd = &pf->stats;
4597         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4598
4599         /* Update hw stats */
4600         ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4601                            pf->stat_offsets_loaded,
4602                            &osd->crc_errors, &nsd->crc_errors);
4603         ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4604                            pf->stat_offsets_loaded,
4605                            &osd->illegal_bytes, &nsd->illegal_bytes);
4606         ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4607                            I40E_GLPRT_GORCL(hw->port),
4608                            pf->stat_offsets_loaded,
4609                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4610         ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4611                            I40E_GLPRT_GOTCL(hw->port),
4612                            pf->stat_offsets_loaded,
4613                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4614         ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4615                            pf->stat_offsets_loaded,
4616                            &osd->eth.rx_discards,
4617                            &nsd->eth.rx_discards);
4618         ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4619                            I40E_GLPRT_UPRCL(hw->port),
4620                            pf->stat_offsets_loaded,
4621                            &osd->eth.rx_unicast,
4622                            &nsd->eth.rx_unicast);
4623         ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4624                            I40E_GLPRT_UPTCL(hw->port),
4625                            pf->stat_offsets_loaded,
4626                            &osd->eth.tx_unicast,
4627                            &nsd->eth.tx_unicast);
4628         ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4629                            I40E_GLPRT_MPRCL(hw->port),
4630                            pf->stat_offsets_loaded,
4631                            &osd->eth.rx_multicast,
4632                            &nsd->eth.rx_multicast);
4633         ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4634                            I40E_GLPRT_MPTCL(hw->port),
4635                            pf->stat_offsets_loaded,
4636                            &osd->eth.tx_multicast,
4637                            &nsd->eth.tx_multicast);
4638         ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4639                            I40E_GLPRT_BPRCL(hw->port),
4640                            pf->stat_offsets_loaded,
4641                            &osd->eth.rx_broadcast,
4642                            &nsd->eth.rx_broadcast);
4643         ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4644                            I40E_GLPRT_BPTCL(hw->port),
4645                            pf->stat_offsets_loaded,
4646                            &osd->eth.tx_broadcast,
4647                            &nsd->eth.tx_broadcast);
4648
4649         ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4650                            pf->stat_offsets_loaded,
4651                            &osd->tx_dropped_link_down,
4652                            &nsd->tx_dropped_link_down);
4653         ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4654                            pf->stat_offsets_loaded,
4655                            &osd->mac_local_faults,
4656                            &nsd->mac_local_faults);
4657         ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4658                            pf->stat_offsets_loaded,
4659                            &osd->mac_remote_faults,
4660                            &nsd->mac_remote_faults);
4661         ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4662                            pf->stat_offsets_loaded,
4663                            &osd->rx_length_errors,
4664                            &nsd->rx_length_errors);
4665
4666         /* Flow control (LFC) stats */
4667         ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4668                            pf->stat_offsets_loaded,
4669                            &osd->link_xon_rx, &nsd->link_xon_rx);
4670         ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4671                            pf->stat_offsets_loaded,
4672                            &osd->link_xon_tx, &nsd->link_xon_tx);
4673         ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4674                            pf->stat_offsets_loaded,
4675                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
4676         ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4677                            pf->stat_offsets_loaded,
4678                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
4679
4680         /* Packet size stats rx */
4681         ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4682                            I40E_GLPRT_PRC64L(hw->port),
4683                            pf->stat_offsets_loaded,
4684                            &osd->rx_size_64, &nsd->rx_size_64);
4685         ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4686                            I40E_GLPRT_PRC127L(hw->port),
4687                            pf->stat_offsets_loaded,
4688                            &osd->rx_size_127, &nsd->rx_size_127);
4689         ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4690                            I40E_GLPRT_PRC255L(hw->port),
4691                            pf->stat_offsets_loaded,
4692                            &osd->rx_size_255, &nsd->rx_size_255);
4693         ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4694                            I40E_GLPRT_PRC511L(hw->port),
4695                            pf->stat_offsets_loaded,
4696                            &osd->rx_size_511, &nsd->rx_size_511);
4697         ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4698                            I40E_GLPRT_PRC1023L(hw->port),
4699                            pf->stat_offsets_loaded,
4700                            &osd->rx_size_1023, &nsd->rx_size_1023);
4701         ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4702                            I40E_GLPRT_PRC1522L(hw->port),
4703                            pf->stat_offsets_loaded,
4704                            &osd->rx_size_1522, &nsd->rx_size_1522);
4705         ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4706                            I40E_GLPRT_PRC9522L(hw->port),
4707                            pf->stat_offsets_loaded,
4708                            &osd->rx_size_big, &nsd->rx_size_big);
4709
4710         /* Packet size stats tx */
4711         ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4712                            I40E_GLPRT_PTC64L(hw->port),
4713                            pf->stat_offsets_loaded,
4714                            &osd->tx_size_64, &nsd->tx_size_64);
4715         ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4716                            I40E_GLPRT_PTC127L(hw->port),
4717                            pf->stat_offsets_loaded,
4718                            &osd->tx_size_127, &nsd->tx_size_127);
4719         ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4720                            I40E_GLPRT_PTC255L(hw->port),
4721                            pf->stat_offsets_loaded,
4722                            &osd->tx_size_255, &nsd->tx_size_255);
4723         ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4724                            I40E_GLPRT_PTC511L(hw->port),
4725                            pf->stat_offsets_loaded,
4726                            &osd->tx_size_511, &nsd->tx_size_511);
4727         ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4728                            I40E_GLPRT_PTC1023L(hw->port),
4729                            pf->stat_offsets_loaded,
4730                            &osd->tx_size_1023, &nsd->tx_size_1023);
4731         ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4732                            I40E_GLPRT_PTC1522L(hw->port),
4733                            pf->stat_offsets_loaded,
4734                            &osd->tx_size_1522, &nsd->tx_size_1522);
4735         ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4736                            I40E_GLPRT_PTC9522L(hw->port),
4737                            pf->stat_offsets_loaded,
4738                            &osd->tx_size_big, &nsd->tx_size_big);
4739
4740         ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4741                            pf->stat_offsets_loaded,
4742                            &osd->rx_undersize, &nsd->rx_undersize);
4743         ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4744                            pf->stat_offsets_loaded,
4745                            &osd->rx_fragments, &nsd->rx_fragments);
4746         ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4747                            pf->stat_offsets_loaded,
4748                            &osd->rx_oversize, &nsd->rx_oversize);
4749         ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4750                            pf->stat_offsets_loaded,
4751                            &osd->rx_jabber, &nsd->rx_jabber);
4752         pf->stat_offsets_loaded = true;
4753         /* End hw stats */
4754
4755         /* Update vsi stats */
4756         ixl_update_vsi_stats(vsi);
4757
4758         for (int i = 0; i < pf->num_vfs; i++) {
4759                 vf = &pf->vfs[i];
4760                 if (vf->vf_flags & VF_FLAG_ENABLED)
4761                         ixl_update_eth_stats(&pf->vfs[i].vsi);
4762         }
4763 }
4764
4765 static int
4766 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf)
4767 {
4768         struct i40e_hw *hw = &pf->hw;
4769         struct ixl_vsi *vsi = &pf->vsi;
4770         device_t dev = pf->dev;
4771         bool is_up = false;
4772         int error = 0;
4773
4774         is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING);
4775
4776         /* Teardown */
4777         if (is_up)
4778                 ixl_stop(pf);
4779         error = i40e_shutdown_lan_hmc(hw);
4780         if (error)
4781                 device_printf(dev,
4782                     "Shutdown LAN HMC failed with code %d\n", error);
4783         ixl_disable_adminq(hw);
4784         ixl_teardown_adminq_msix(pf);
4785         error = i40e_shutdown_adminq(hw);
4786         if (error)
4787                 device_printf(dev,
4788                     "Shutdown Admin queue failed with code %d\n", error);
4789
4790         /* Setup */
4791         error = i40e_init_adminq(hw);
4792         if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
4793                 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
4794                     error);
4795         }
4796         error = ixl_setup_adminq_msix(pf);
4797         if (error) {
4798                 device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
4799                     error);
4800         }
4801         ixl_configure_intr0_msix(pf);
4802         ixl_enable_adminq(hw);
4803         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4804             hw->func_caps.num_rx_qp, 0, 0);
4805         if (error) {
4806                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
4807         }
4808         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4809         if (error) {
4810                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
4811         }
4812         if (is_up)
4813                 ixl_init(pf);
4814
4815         return (0);
4816 }
4817
4818 static void
4819 ixl_handle_empr_reset(struct ixl_pf *pf)
4820 {
4821         struct i40e_hw *hw = &pf->hw;
4822         device_t dev = pf->dev;
4823         int count = 0;
4824         u32 reg;
4825
4826         /* Typically finishes within 3-4 seconds */
4827         while (count++ < 100) {
4828                 reg = rd32(hw, I40E_GLGEN_RSTAT)
4829                     & I40E_GLGEN_RSTAT_DEVSTATE_MASK;
4830                 if (reg)
4831                         i40e_msec_delay(100);
4832                 else
4833                         break;
4834         }
4835 #ifdef IXL_DEBUG
4836         // Reset-related
4837         device_printf(dev, "EMPR reset wait count: %d\n", count);
4838 #endif
4839
4840         device_printf(dev, "Rebuilding driver state...\n");
4841         ixl_rebuild_hw_structs_after_reset(pf);
4842         device_printf(dev, "Rebuilding driver state done.\n");
4843
4844         atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
4845 }
4846
4847 /*
4848 ** Tasklet handler for MSIX Adminq interrupts
4849 **  - do outside interrupt since it might sleep
4850 */
4851 static void
4852 ixl_do_adminq(void *context, int pending)
4853 {
4854         struct ixl_pf                   *pf = context;
4855         struct i40e_hw                  *hw = &pf->hw;
4856         struct i40e_arq_event_info      event;
4857         i40e_status                     ret;
4858         device_t                        dev = pf->dev;
4859         u32                             loop = 0;
4860         u16                             opcode, result;
4861
4862         if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
4863                 /* Flag cleared at end of this function */
4864                 ixl_handle_empr_reset(pf);
4865                 return;
4866         }
4867
4868         /* Admin Queue handling */
4869         event.buf_len = IXL_AQ_BUF_SZ;
4870         event.msg_buf = malloc(event.buf_len,
4871             M_DEVBUF, M_NOWAIT | M_ZERO);
4872         if (!event.msg_buf) {
4873                 device_printf(dev, "%s: Unable to allocate memory for Admin"
4874                     " Queue event!\n", __func__);
4875                 return;
4876         }
4877
4878         IXL_PF_LOCK(pf);
4879         /* clean and process any events */
4880         do {
4881                 ret = i40e_clean_arq_element(hw, &event, &result);
4882                 if (ret)
4883                         break;
4884                 opcode = LE16_TO_CPU(event.desc.opcode);
4885 #ifdef IXL_DEBUG
4886                 device_printf(dev, "%s: Admin Queue event: %#06x\n", __func__,
4887                     opcode);
4888 #endif
4889                 switch (opcode) {
4890                 case i40e_aqc_opc_get_link_status:
4891                         ixl_link_event(pf, &event);
4892                         break;
4893                 case i40e_aqc_opc_send_msg_to_pf:
4894 #ifdef PCI_IOV
4895                         ixl_handle_vf_msg(pf, &event);
4896 #endif
4897                         break;
4898                 case i40e_aqc_opc_event_lan_overflow:
4899                 default:
4900                         break;
4901                 }
4902
4903         } while (result && (loop++ < IXL_ADM_LIMIT));
4904
4905         free(event.msg_buf, M_DEVBUF);
4906
4907         /*
4908          * If there are still messages to process, reschedule ourselves.
4909          * Otherwise, re-enable our interrupt and go to sleep.
4910          */
4911         if (result > 0)
4912                 taskqueue_enqueue(pf->tq, &pf->adminq);
4913         else
4914                 ixl_enable_adminq(hw);
4915
4916         IXL_PF_UNLOCK(pf);
4917 }
4918
4919 /**
4920  * Update VSI-specific ethernet statistics counters.
4921  **/
4922 void
4923 ixl_update_eth_stats(struct ixl_vsi *vsi)
4924 {
4925         struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4926         struct i40e_hw *hw = &pf->hw;
4927         struct i40e_eth_stats *es;
4928         struct i40e_eth_stats *oes;
4929         struct i40e_hw_port_stats *nsd;
4930         u16 stat_idx = vsi->info.stat_counter_idx;
4931
4932         es = &vsi->eth_stats;
4933         oes = &vsi->eth_stats_offsets;
4934         nsd = &pf->stats;
4935
4936         /* Gather up the stats that the hw collects */
4937         ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4938                            vsi->stat_offsets_loaded,
4939                            &oes->tx_errors, &es->tx_errors);
4940         ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4941                            vsi->stat_offsets_loaded,
4942                            &oes->rx_discards, &es->rx_discards);
4943
4944         ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4945                            I40E_GLV_GORCL(stat_idx),
4946                            vsi->stat_offsets_loaded,
4947                            &oes->rx_bytes, &es->rx_bytes);
4948         ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4949                            I40E_GLV_UPRCL(stat_idx),
4950                            vsi->stat_offsets_loaded,
4951                            &oes->rx_unicast, &es->rx_unicast);
4952         ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4953                            I40E_GLV_MPRCL(stat_idx),
4954                            vsi->stat_offsets_loaded,
4955                            &oes->rx_multicast, &es->rx_multicast);
4956         ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4957                            I40E_GLV_BPRCL(stat_idx),
4958                            vsi->stat_offsets_loaded,
4959                            &oes->rx_broadcast, &es->rx_broadcast);
4960
4961         ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4962                            I40E_GLV_GOTCL(stat_idx),
4963                            vsi->stat_offsets_loaded,
4964                            &oes->tx_bytes, &es->tx_bytes);
4965         ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4966                            I40E_GLV_UPTCL(stat_idx),
4967                            vsi->stat_offsets_loaded,
4968                            &oes->tx_unicast, &es->tx_unicast);
4969         ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4970                            I40E_GLV_MPTCL(stat_idx),
4971                            vsi->stat_offsets_loaded,
4972                            &oes->tx_multicast, &es->tx_multicast);
4973         ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4974                            I40E_GLV_BPTCL(stat_idx),
4975                            vsi->stat_offsets_loaded,
4976                            &oes->tx_broadcast, &es->tx_broadcast);
4977         vsi->stat_offsets_loaded = true;
4978 }
4979
4980 static void
4981 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4982 {
4983         struct ixl_pf           *pf;
4984         struct ifnet            *ifp;
4985         struct i40e_eth_stats   *es;
4986         u64                     tx_discards;
4987
4988         struct i40e_hw_port_stats *nsd;
4989
4990         pf = vsi->back;
4991         ifp = vsi->ifp;
4992         es = &vsi->eth_stats;
4993         nsd = &pf->stats;
4994
4995         ixl_update_eth_stats(vsi);
4996
4997         tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4998         for (int i = 0; i < vsi->num_queues; i++)
4999                 tx_discards += vsi->queues[i].txr.br->br_drops;
5000
5001         /* Update ifnet stats */
5002         IXL_SET_IPACKETS(vsi, es->rx_unicast +
5003                            es->rx_multicast +
5004                            es->rx_broadcast);
5005         IXL_SET_OPACKETS(vsi, es->tx_unicast +
5006                            es->tx_multicast +
5007                            es->tx_broadcast);
5008         IXL_SET_IBYTES(vsi, es->rx_bytes);
5009         IXL_SET_OBYTES(vsi, es->tx_bytes);
5010         IXL_SET_IMCASTS(vsi, es->rx_multicast);
5011         IXL_SET_OMCASTS(vsi, es->tx_multicast);
5012
5013         IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
5014             nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
5015             nsd->rx_jabber);
5016         IXL_SET_OERRORS(vsi, es->tx_errors);
5017         IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
5018         IXL_SET_OQDROPS(vsi, tx_discards);
5019         IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
5020         IXL_SET_COLLISIONS(vsi, 0);
5021 }
5022
5023 /**
5024  * Reset all of the stats for the given pf
5025  **/
5026 void ixl_pf_reset_stats(struct ixl_pf *pf)
5027 {
5028         bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
5029         bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
5030         pf->stat_offsets_loaded = false;
5031 }
5032
5033 /**
5034  * Resets all stats of the given vsi
5035  **/
5036 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
5037 {
5038         bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
5039         bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
5040         vsi->stat_offsets_loaded = false;
5041 }
5042
5043 /**
5044  * Read and update a 48 bit stat from the hw
5045  *
5046  * Since the device stats are not reset at PFReset, they likely will not
5047  * be zeroed when the driver starts.  We'll save the first values read
5048  * and use them as offsets to be subtracted from the raw values in order
5049  * to report stats that count from zero.
5050  **/
5051 static void
5052 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
5053         bool offset_loaded, u64 *offset, u64 *stat)
5054 {
5055         u64 new_data;
5056
5057 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
5058         new_data = rd64(hw, loreg);
5059 #else
5060         /*
5061          * Use two rd32's instead of one rd64; FreeBSD versions before
5062          * 10 don't support 8 byte bus reads/writes.
5063          */
5064         new_data = rd32(hw, loreg);
5065         new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
5066 #endif
5067
5068         if (!offset_loaded)
5069                 *offset = new_data;
5070         if (new_data >= *offset)
5071                 *stat = new_data - *offset;
5072         else
5073                 *stat = (new_data + ((u64)1 << 48)) - *offset;
5074         *stat &= 0xFFFFFFFFFFFFULL;
5075 }
5076
5077 /**
5078  * Read and update a 32 bit stat from the hw
5079  **/
5080 static void
5081 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
5082         bool offset_loaded, u64 *offset, u64 *stat)
5083 {
5084         u32 new_data;
5085
5086         new_data = rd32(hw, reg);
5087         if (!offset_loaded)
5088                 *offset = new_data;
5089         if (new_data >= *offset)
5090                 *stat = (u32)(new_data - *offset);
5091         else
5092                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
5093 }
5094
5095 static void
5096 ixl_add_device_sysctls(struct ixl_pf *pf)
5097 {
5098         device_t dev = pf->dev;
5099
5100         /* Set up sysctls */
5101         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5102             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5103             OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
5104             pf, 0, ixl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC);
5105
5106         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5107             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5108             OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
5109             pf, 0, ixl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE);
5110
5111         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5112             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5113             OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
5114             pf, 0, ixl_current_speed, "A", "Current Port Speed");
5115
5116         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5117             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5118             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
5119             pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
5120
5121 #if 0
5122         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5123             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5124             OID_AUTO, "rx_itr", CTLFLAG_RW,
5125             &ixl_rx_itr, 0, "RX ITR");
5126
5127         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5128             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5129             OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
5130             &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
5131
5132         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5133             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5134             OID_AUTO, "tx_itr", CTLFLAG_RW,
5135             &ixl_tx_itr, 0, "TX ITR");
5136
5137         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5138             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5139             OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
5140             &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
5141 #endif
5142
5143 #ifdef IXL_DEBUG_SYSCTL
5144         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5145             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5146             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
5147             ixl_debug_info, "I", "Debug Information");
5148
5149         /* Shared-code debug message level */
5150         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5151             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5152             OID_AUTO, "debug_mask", CTLFLAG_RW,
5153             &pf->hw.debug_mask, 0, "Debug Message Level");
5154
5155         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5156             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5157             OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
5158             pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS);
5159
5160         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5161             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5162             OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
5163             pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
5164
5165         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5166             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5167             OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
5168             pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
5169
5170         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5171             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5172             OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
5173             pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
5174
5175         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5176             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5177             OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
5178             pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
5179
5180 #ifdef PCI_IOV
5181         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5182             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5183             OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
5184             0, "PF/VF Virtual Channel debug level");
5185 #endif
5186 #endif
5187 }
5188
5189 /*
5190 ** Set flow control using sysctl:
5191 **      0 - off
5192 **      1 - rx pause
5193 **      2 - tx pause
5194 **      3 - full
5195 */
5196 static int
5197 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
5198 {
5199         /*
5200          * TODO: ensure tx CRC by hardware should be enabled
5201          * if tx flow control is enabled.
5202          * ^ N/A for 40G ports
5203          */
5204         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5205         struct i40e_hw *hw = &pf->hw;
5206         device_t dev = pf->dev;
5207         int requested_fc, error = 0;
5208         enum i40e_status_code aq_error = 0;
5209         u8 fc_aq_err = 0;
5210
5211         /* Get request */
5212         requested_fc = pf->fc;
5213         error = sysctl_handle_int(oidp, &requested_fc, 0, req);
5214         if ((error) || (req->newptr == NULL))
5215                 return (error);
5216         if (requested_fc < 0 || requested_fc > 3) {
5217                 device_printf(dev,
5218                     "Invalid fc mode; valid modes are 0 through 3\n");
5219                 return (EINVAL);
5220         }
5221
5222         /* Set fc ability for port */
5223         hw->fc.requested_mode = requested_fc;
5224         aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
5225         if (aq_error) {
5226                 device_printf(dev,
5227                     "%s: Error setting new fc mode %d; fc_err %#x\n",
5228                     __func__, aq_error, fc_aq_err);
5229                 return (EIO);
5230         }
5231         pf->fc = requested_fc;
5232
5233         /* Get new link state */
5234         i40e_msec_delay(250);
5235         hw->phy.get_link_info = TRUE;
5236         i40e_get_link_status(hw, &pf->link_up);
5237
5238         return (0);
5239 }
5240
5241 static int
5242 ixl_current_speed(SYSCTL_HANDLER_ARGS)
5243 {
5244         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5245         struct i40e_hw *hw = &pf->hw;
5246         int error = 0, index = 0;
5247
5248         char *speeds[] = {
5249                 "Unknown",
5250                 "100M",
5251                 "1G",
5252                 "10G",
5253                 "40G",
5254                 "20G"
5255         };
5256
5257         ixl_update_link_status(pf);
5258
5259         switch (hw->phy.link_info.link_speed) {
5260         case I40E_LINK_SPEED_100MB:
5261                 index = 1;
5262                 break;
5263         case I40E_LINK_SPEED_1GB:
5264                 index = 2;
5265                 break;
5266         case I40E_LINK_SPEED_10GB:
5267                 index = 3;
5268                 break;
5269         case I40E_LINK_SPEED_40GB:
5270                 index = 4;
5271                 break;
5272         case I40E_LINK_SPEED_20GB:
5273                 index = 5;
5274                 break;
5275         case I40E_LINK_SPEED_UNKNOWN:
5276         default:
5277                 index = 0;
5278                 break;
5279         }
5280
5281         error = sysctl_handle_string(oidp, speeds[index],
5282             strlen(speeds[index]), req);
5283         return (error);
5284 }
5285
5286 static int
5287 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
5288 {
5289         struct i40e_hw *hw = &pf->hw;
5290         device_t dev = pf->dev;
5291         struct i40e_aq_get_phy_abilities_resp abilities;
5292         struct i40e_aq_set_phy_config config;
5293         enum i40e_status_code aq_error = 0;
5294
5295         /* Get current capability information */
5296         aq_error = i40e_aq_get_phy_capabilities(hw,
5297             FALSE, FALSE, &abilities, NULL);
5298         if (aq_error) {
5299                 device_printf(dev,
5300                     "%s: Error getting phy capabilities %d,"
5301                     " aq error: %d\n", __func__, aq_error,
5302                     hw->aq.asq_last_status);
5303                 return (EAGAIN);
5304         }
5305
5306         /* Prepare new config */
5307         bzero(&config, sizeof(config));
5308         config.phy_type = abilities.phy_type;
5309         config.abilities = abilities.abilities
5310             | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
5311         config.eee_capability = abilities.eee_capability;
5312         config.eeer = abilities.eeer_val;
5313         config.low_power_ctrl = abilities.d3_lpan;
5314         /* Translate into aq cmd link_speed */
5315         if (speeds & 0x10)
5316                 config.link_speed |= I40E_LINK_SPEED_40GB;
5317         if (speeds & 0x8)
5318                 config.link_speed |= I40E_LINK_SPEED_20GB;
5319         if (speeds & 0x4)
5320                 config.link_speed |= I40E_LINK_SPEED_10GB;
5321         if (speeds & 0x2)
5322                 config.link_speed |= I40E_LINK_SPEED_1GB;
5323         if (speeds & 0x1)
5324                 config.link_speed |= I40E_LINK_SPEED_100MB;
5325
5326         /* Do aq command & restart link */
5327         aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
5328         if (aq_error) {
5329                 device_printf(dev,
5330                     "%s: Error setting new phy config %d,"
5331                     " aq error: %d\n", __func__, aq_error,
5332                     hw->aq.asq_last_status);
5333                 return (EAGAIN);
5334         }
5335
5336         /*
5337         ** This seems a bit heavy handed, but we
5338         ** need to get a reinit on some devices
5339         */
5340         IXL_PF_LOCK(pf);
5341         ixl_stop_locked(pf);
5342         ixl_init_locked(pf);
5343         IXL_PF_UNLOCK(pf);
5344
5345         return (0);
5346 }
5347
5348 /*
5349 ** Control link advertise speed:
5350 **      Flags:
5351 **       0x1 - advertise 100 Mb
5352 **       0x2 - advertise 1G
5353 **       0x4 - advertise 10G
5354 **       0x8 - advertise 20G
5355 **      0x10 - advertise 40G
5356 **
5357 **      Set to 0 to disable link
5358 */
5359 static int
5360 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
5361 {
5362         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5363         struct i40e_hw *hw = &pf->hw;
5364         device_t dev = pf->dev;
5365         int requested_ls = 0;
5366         int error = 0;
5367
5368         /* Read in new mode */
5369         requested_ls = pf->advertised_speed;
5370         error = sysctl_handle_int(oidp, &requested_ls, 0, req);
5371         if ((error) || (req->newptr == NULL))
5372                 return (error);
5373         /* Check for sane value */
5374         if (requested_ls > 0x10) {
5375                 device_printf(dev, "Invalid advertised speed; "
5376                     "valid modes are 0x1 through 0x10\n");
5377                 return (EINVAL);
5378         }
5379         /* Then check for validity based on adapter type */
5380         switch (hw->device_id) {
5381         case I40E_DEV_ID_10G_BASE_T:
5382         case I40E_DEV_ID_10G_BASE_T4:
5383                 /* BaseT */
5384                 if (requested_ls & ~(0x7)) {
5385                         device_printf(dev,
5386                             "Only 100M/1G/10G speeds supported on this device.\n");
5387                         return (EINVAL);
5388                 }
5389                 break;
5390         case I40E_DEV_ID_20G_KR2:
5391         case I40E_DEV_ID_20G_KR2_A:
5392                 /* 20G */
5393                 if (requested_ls & ~(0xE)) {
5394                         device_printf(dev,
5395                             "Only 1G/10G/20G speeds supported on this device.\n");
5396                         return (EINVAL);
5397                 }
5398                 break;
5399         case I40E_DEV_ID_KX_B:
5400         case I40E_DEV_ID_QSFP_A:
5401         case I40E_DEV_ID_QSFP_B:
5402                 /* 40G */
5403                 if (requested_ls & ~(0x10)) {
5404                         device_printf(dev,
5405                             "Only 40G speeds supported on this device.\n");
5406                         return (EINVAL);
5407                 }
5408                 break;
5409         default:
5410                 /* 10G (1G) */
5411                 if (requested_ls & ~(0x6)) {
5412                         device_printf(dev,
5413                             "Only 1/10Gbs speeds are supported on this device.\n");
5414                         return (EINVAL);
5415                 }
5416                 break;
5417         }
5418
5419         /* Exit if no change */
5420         if (pf->advertised_speed == requested_ls)
5421                 return (0);
5422
5423         error = ixl_set_advertised_speeds(pf, requested_ls);
5424         if (error)
5425                 return (error);
5426
5427         pf->advertised_speed = requested_ls;
5428         ixl_update_link_status(pf);
5429         return (0);
5430 }
5431
5432 /*
5433 ** Get the width and transaction speed of
5434 ** the bus this adapter is plugged into.
5435 */
5436 static u16
5437 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
5438 {
5439         u16                     link;
5440         u32                     offset;
5441
5442         /* Get the PCI Express Capabilities offset */
5443         pci_find_cap(dev, PCIY_EXPRESS, &offset);
5444
5445         /* ...and read the Link Status Register */
5446         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
5447
5448         switch (link & I40E_PCI_LINK_WIDTH) {
5449         case I40E_PCI_LINK_WIDTH_1:
5450                 hw->bus.width = i40e_bus_width_pcie_x1;
5451                 break;
5452         case I40E_PCI_LINK_WIDTH_2:
5453                 hw->bus.width = i40e_bus_width_pcie_x2;
5454                 break;
5455         case I40E_PCI_LINK_WIDTH_4:
5456                 hw->bus.width = i40e_bus_width_pcie_x4;
5457                 break;
5458         case I40E_PCI_LINK_WIDTH_8:
5459                 hw->bus.width = i40e_bus_width_pcie_x8;
5460                 break;
5461         default:
5462                 hw->bus.width = i40e_bus_width_unknown;
5463                 break;
5464         }
5465
5466         switch (link & I40E_PCI_LINK_SPEED) {
5467         case I40E_PCI_LINK_SPEED_2500:
5468                 hw->bus.speed = i40e_bus_speed_2500;
5469                 break;
5470         case I40E_PCI_LINK_SPEED_5000:
5471                 hw->bus.speed = i40e_bus_speed_5000;
5472                 break;
5473         case I40E_PCI_LINK_SPEED_8000:
5474                 hw->bus.speed = i40e_bus_speed_8000;
5475                 break;
5476         default:
5477                 hw->bus.speed = i40e_bus_speed_unknown;
5478                 break;
5479         }
5480
5481         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
5482             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
5483             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
5484             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
5485             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
5486             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
5487             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
5488             ("Unknown"));
5489
5490         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
5491             (hw->bus.speed < i40e_bus_speed_8000)) {
5492                 device_printf(dev, "PCI-Express bandwidth available"
5493                     " for this device\n     may be insufficient for"
5494                     " optimal performance.\n");
5495                 device_printf(dev, "For expected performance a x8 "
5496                     "PCIE Gen3 slot is required.\n");
5497         }
5498
5499         return (link);
5500 }
5501
5502 static int
5503 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
5504 {
5505         struct ixl_pf   *pf = (struct ixl_pf *)arg1;
5506         struct i40e_hw  *hw = &pf->hw;
5507         struct sbuf     *sbuf;
5508
5509         sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5510         ixl_nvm_version_str(hw, sbuf);
5511         sbuf_finish(sbuf);
5512         sbuf_delete(sbuf);
5513
5514         return 0;
5515 }
5516
5517 #ifdef IXL_DEBUG
5518 static void
5519 ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma)
5520 {
5521         if ((nvma->command == I40E_NVM_READ) &&
5522             ((nvma->config & 0xFF) == 0xF) &&
5523             (((nvma->config & 0xF00) >> 8) == 0xF) &&
5524             (nvma->offset == 0) &&
5525             (nvma->data_size == 1)) {
5526                 // device_printf(dev, "- Get Driver Status Command\n");
5527         }
5528         else if (nvma->command == I40E_NVM_READ) {
5529
5530         }
5531         else {
5532                 switch (nvma->command) {
5533                 case 0xB:
5534                         device_printf(dev, "- command: I40E_NVM_READ\n");
5535                         break;
5536                 case 0xC:
5537                         device_printf(dev, "- command: I40E_NVM_WRITE\n");
5538                         break;
5539                 default:
5540                         device_printf(dev, "- command: unknown 0x%08x\n", nvma->command);
5541                         break;
5542                 }
5543
5544                 device_printf(dev, "- config (ptr)  : 0x%02x\n", nvma->config & 0xFF);
5545                 device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8);
5546                 device_printf(dev, "- offset : 0x%08x\n", nvma->offset);
5547                 device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size);
5548         }
5549 }
5550 #endif
5551
5552 static int
5553 ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd)
5554 {
5555         struct i40e_hw *hw = &pf->hw;
5556         struct i40e_nvm_access *nvma;
5557         device_t dev = pf->dev;
5558         enum i40e_status_code status = 0;
5559         int perrno;
5560
5561         DEBUGFUNC("ixl_handle_nvmupd_cmd");
5562
5563         /* Sanity checks */
5564         if (ifd->ifd_len < sizeof(struct i40e_nvm_access) ||
5565             ifd->ifd_data == NULL) {
5566                 device_printf(dev, "%s: incorrect ifdrv length or data pointer\n",
5567                     __func__);
5568                 device_printf(dev, "%s: ifdrv length: %lu, sizeof(struct i40e_nvm_access): %lu\n",
5569                     __func__, ifd->ifd_len, sizeof(struct i40e_nvm_access));
5570                 device_printf(dev, "%s: data pointer: %p\n", __func__,
5571                     ifd->ifd_data);
5572                 return (EINVAL);
5573         }
5574
5575         nvma = (struct i40e_nvm_access *)ifd->ifd_data;
5576
5577 #ifdef IXL_DEBUG
5578         ixl_print_nvm_cmd(dev, nvma);
5579 #endif
5580
5581         if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
5582                 int count = 0;
5583                 while (count++ < 100) {
5584                         i40e_msec_delay(100);
5585                         if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING))
5586                                 break;
5587                 }
5588         }
5589
5590         if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) {
5591                 IXL_PF_LOCK(pf);
5592                 status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno);
5593                 IXL_PF_UNLOCK(pf);
5594         } else {
5595                 perrno = -EBUSY;
5596         }
5597
5598         if (status)
5599                 device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n",
5600                     status, perrno);
5601
5602         /*
5603          * -EPERM is actually ERESTART, which the kernel interprets as it needing
5604          * to run this ioctl again. So use -EACCES for -EPERM instead.
5605          */
5606         if (perrno == -EPERM)
5607                 return (-EACCES);
5608         else
5609                 return (perrno);
5610 }
5611
5612 #ifdef IXL_DEBUG_SYSCTL
5613 static int
5614 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
5615 {
5616         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5617         struct i40e_hw *hw = &pf->hw;
5618         struct i40e_link_status link_status;
5619         char buf[512];
5620
5621         enum i40e_status_code aq_error = 0;
5622
5623         aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
5624         if (aq_error) {
5625                 printf("i40e_aq_get_link_info() error %d\n", aq_error);
5626                 return (EPERM);
5627         }
5628
5629         sprintf(buf, "\n"
5630             "PHY Type : %#04x\n"
5631             "Speed    : %#04x\n"
5632             "Link info: %#04x\n"
5633             "AN info  : %#04x\n"
5634             "Ext info : %#04x\n"
5635             "Max Frame: %d\n"
5636             "Pacing   : %#04x\n"
5637             "CRC En?  : %d",
5638             link_status.phy_type, link_status.link_speed,
5639             link_status.link_info, link_status.an_info,
5640             link_status.ext_info, link_status.max_frame_size,
5641             link_status.pacing, link_status.crc_enable);
5642
5643         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5644 }
5645
5646 static int
5647 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
5648 {
5649         struct ixl_pf           *pf = (struct ixl_pf *)arg1;
5650         struct i40e_hw          *hw = &pf->hw;
5651         char                    buf[512];
5652         enum i40e_status_code   aq_error = 0;
5653
5654         struct i40e_aq_get_phy_abilities_resp abilities;
5655
5656         aq_error = i40e_aq_get_phy_capabilities(hw,
5657             TRUE, FALSE, &abilities, NULL);
5658         if (aq_error) {
5659                 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
5660                 return (EPERM);
5661         }
5662
5663         sprintf(buf, "\n"
5664             "PHY Type : %#010x\n"
5665             "Speed    : %#04x\n"
5666             "Abilities: %#04x\n"
5667             "EEE cap  : %#06x\n"
5668             "EEER reg : %#010x\n"
5669             "D3 Lpan  : %#04x",
5670             abilities.phy_type, abilities.link_speed,
5671             abilities.abilities, abilities.eee_capability,
5672             abilities.eeer_val, abilities.d3_lpan);
5673
5674         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5675 }
5676
5677 static int
5678 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5679 {
5680         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5681         struct ixl_vsi *vsi = &pf->vsi;
5682         struct ixl_mac_filter *f;
5683         char *buf, *buf_i;
5684
5685         int error = 0;
5686         int ftl_len = 0;
5687         int ftl_counter = 0;
5688         int buf_len = 0;
5689         int entry_len = 42;
5690
5691         SLIST_FOREACH(f, &vsi->ftl, next) {
5692                 ftl_len++;
5693         }
5694
5695         if (ftl_len < 1) {
5696                 sysctl_handle_string(oidp, "(none)", 6, req);
5697                 return (0);
5698         }
5699
5700         buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5701         buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5702
5703         sprintf(buf_i++, "\n");
5704         SLIST_FOREACH(f, &vsi->ftl, next) {
5705                 sprintf(buf_i,
5706                     MAC_FORMAT ", vlan %4d, flags %#06x",
5707                     MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5708                 buf_i += entry_len;
5709                 /* don't print '\n' for last entry */
5710                 if (++ftl_counter != ftl_len) {
5711                         sprintf(buf_i, "\n");
5712                         buf_i++;
5713                 }
5714         }
5715
5716         error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5717         if (error)
5718                 printf("sysctl error: %d\n", error);
5719         free(buf, M_DEVBUF);
5720         return error;
5721 }
5722
5723 #define IXL_SW_RES_SIZE 0x14
5724 static int
5725 ixl_res_alloc_cmp(const void *a, const void *b)
5726 {
5727         const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5728         one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5729         two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5730
5731         return ((int)one->resource_type - (int)two->resource_type);
5732 }
5733
5734 /*
5735  * Longest string length: 25
5736  */
5737 static char *
5738 ixl_switch_res_type_string(u8 type)
5739 {
5740         static char * ixl_switch_res_type_strings[0x14] = {
5741                 "VEB",
5742                 "VSI",
5743                 "Perfect Match MAC address",
5744                 "S-tag",
5745                 "(Reserved)",
5746                 "Multicast hash entry",
5747                 "Unicast hash entry",
5748                 "VLAN",
5749                 "VSI List entry",
5750                 "(Reserved)",
5751                 "VLAN Statistic Pool",
5752                 "Mirror Rule",
5753                 "Queue Set",
5754                 "Inner VLAN Forward filter",
5755                 "(Reserved)",
5756                 "Inner MAC",
5757                 "IP",
5758                 "GRE/VN1 Key",
5759                 "VN2 Key",
5760                 "Tunneling Port"
5761         };
5762
5763         if (type < 0x14)
5764                 return ixl_switch_res_type_strings[type];
5765         else
5766                 return "(Reserved)";
5767 }
5768
5769 static int
5770 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5771 {
5772         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5773         struct i40e_hw *hw = &pf->hw;
5774         device_t dev = pf->dev;
5775         struct sbuf *buf;
5776         int error = 0;
5777
5778         u8 num_entries;
5779         struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5780
5781         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5782         if (!buf) {
5783                 device_printf(dev, "Could not allocate sbuf for output.\n");
5784                 return (ENOMEM);
5785         }
5786
5787         bzero(resp, sizeof(resp));
5788         error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5789                                 resp,
5790                                 IXL_SW_RES_SIZE,
5791                                 NULL);
5792         if (error) {
5793                 device_printf(dev,
5794                     "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5795                     __func__, error, hw->aq.asq_last_status);
5796                 sbuf_delete(buf);
5797                 return error;
5798         }
5799
5800         /* Sort entries by type for display */
5801         qsort(resp, num_entries,
5802             sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5803             &ixl_res_alloc_cmp);
5804
5805         sbuf_cat(buf, "\n");
5806         sbuf_printf(buf, "# of entries: %d\n", num_entries);
5807         sbuf_printf(buf,
5808             "                     Type | Guaranteed | Total | Used   | Un-allocated\n"
5809             "                          | (this)     | (all) | (this) | (all)       \n");
5810         for (int i = 0; i < num_entries; i++) {
5811                 sbuf_printf(buf,
5812                     "%25s | %10d   %5d   %6d   %12d",
5813                     ixl_switch_res_type_string(resp[i].resource_type),
5814                     resp[i].guaranteed,
5815                     resp[i].total,
5816                     resp[i].used,
5817                     resp[i].total_unalloced);
5818                 if (i < num_entries - 1)
5819                         sbuf_cat(buf, "\n");
5820         }
5821
5822         error = sbuf_finish(buf);
5823         if (error)
5824                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5825
5826         sbuf_delete(buf);
5827         return error;
5828 }
5829
5830 /*
5831 ** Caller must init and delete sbuf; this function will clear and
5832 ** finish it for caller.
5833 **
5834 ** XXX: Cannot use the SEID for this, since there is no longer a
5835 ** fixed mapping between SEID and element type.
5836 */
5837 static char *
5838 ixl_switch_element_string(struct sbuf *s,
5839     struct i40e_aqc_switch_config_element_resp *element)
5840 {
5841         sbuf_clear(s);
5842
5843         switch (element->element_type) {
5844         case I40E_AQ_SW_ELEM_TYPE_MAC:
5845                 sbuf_printf(s, "MAC %3d", element->element_info);
5846                 break;
5847         case I40E_AQ_SW_ELEM_TYPE_PF:
5848                 sbuf_printf(s, "PF  %3d", element->element_info);
5849                 break;
5850         case I40E_AQ_SW_ELEM_TYPE_VF:
5851                 sbuf_printf(s, "VF  %3d", element->element_info);
5852                 break;
5853         case I40E_AQ_SW_ELEM_TYPE_EMP:
5854                 sbuf_cat(s, "EMP");
5855                 break;
5856         case I40E_AQ_SW_ELEM_TYPE_BMC:
5857                 sbuf_cat(s, "BMC");
5858                 break;
5859         case I40E_AQ_SW_ELEM_TYPE_PV:
5860                 sbuf_cat(s, "PV");
5861                 break;
5862         case I40E_AQ_SW_ELEM_TYPE_VEB:
5863                 sbuf_cat(s, "VEB");
5864                 break;
5865         case I40E_AQ_SW_ELEM_TYPE_PA:
5866                 sbuf_cat(s, "PA");
5867                 break;
5868         case I40E_AQ_SW_ELEM_TYPE_VSI:
5869                 sbuf_printf(s, "VSI %3d", element->element_info);
5870                 break;
5871         default:
5872                 sbuf_cat(s, "?");
5873                 break;
5874         }
5875
5876         sbuf_finish(s);
5877         return sbuf_data(s);
5878 }
5879
5880 static int
5881 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5882 {
5883         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5884         struct i40e_hw *hw = &pf->hw;
5885         device_t dev = pf->dev;
5886         struct sbuf *buf;
5887         struct sbuf *nmbuf;
5888         int error = 0;
5889         u16 next = 0;
5890         u8 aq_buf[I40E_AQ_LARGE_BUF];
5891
5892         struct i40e_aqc_get_switch_config_resp *sw_config;
5893         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5894
5895         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5896         if (!buf) {
5897                 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5898                 return (ENOMEM);
5899         }
5900
5901         error = i40e_aq_get_switch_config(hw, sw_config,
5902             sizeof(aq_buf), &next, NULL);
5903         if (error) {
5904                 device_printf(dev,
5905                     "%s: aq_get_switch_config() error %d, aq error %d\n",
5906                     __func__, error, hw->aq.asq_last_status);
5907                 sbuf_delete(buf);
5908                 return error;
5909         }
5910         if (next)
5911                 device_printf(dev, "%s: TODO: get more config with SEID %d\n",
5912                     __func__, next);
5913
5914         nmbuf = sbuf_new_auto();
5915         if (!nmbuf) {
5916                 device_printf(dev, "Could not allocate sbuf for name output.\n");
5917                 sbuf_delete(buf);
5918                 return (ENOMEM);
5919         }
5920
5921         sbuf_cat(buf, "\n");
5922         // Assuming <= 255 elements in switch
5923         sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported);
5924         sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total);
5925         /* Exclude:
5926         ** Revision -- all elements are revision 1 for now
5927         */
5928         sbuf_printf(buf,
5929             "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5930             "                |          |          | (uplink)\n");
5931         for (int i = 0; i < sw_config->header.num_reported; i++) {
5932                 // "%4d (%8s) | %8s   %8s   %#8x",
5933                 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5934                 sbuf_cat(buf, " ");
5935                 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5936                     &sw_config->element[i]));
5937                 sbuf_cat(buf, " | ");
5938                 sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid);
5939                 sbuf_cat(buf, "   ");
5940                 sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid);
5941                 sbuf_cat(buf, "   ");
5942                 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5943                 if (i < sw_config->header.num_reported - 1)
5944                         sbuf_cat(buf, "\n");
5945         }
5946         sbuf_delete(nmbuf);
5947
5948         error = sbuf_finish(buf);
5949         if (error)
5950                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5951
5952         sbuf_delete(buf);
5953
5954         return (error);
5955 }
5956
5957 static int
5958 ixl_debug_info(SYSCTL_HANDLER_ARGS)
5959 {
5960         struct ixl_pf   *pf;
5961         int             error, input = 0;
5962
5963         error = sysctl_handle_int(oidp, &input, 0, req);
5964
5965         if (error || !req->newptr)
5966                 return (error);
5967
5968         if (input == 1) {
5969                 pf = (struct ixl_pf *)arg1;
5970                 ixl_print_debug_info(pf);
5971         }
5972
5973         return (error);
5974 }
5975
5976 static void
5977 ixl_print_debug_info(struct ixl_pf *pf)
5978 {
5979         struct i40e_hw          *hw = &pf->hw;
5980         struct ixl_vsi          *vsi = &pf->vsi;
5981         struct ixl_queue        *que = vsi->queues;
5982         struct rx_ring          *rxr = &que->rxr;
5983         struct tx_ring          *txr = &que->txr;
5984         u32                     reg;
5985
5986
5987         printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
5988         printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
5989         printf("RX next check = %x\n", rxr->next_check);
5990         printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
5991         printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
5992         printf("TX desc avail = %x\n", txr->avail);
5993
5994         reg = rd32(hw, I40E_GLV_GORCL(0xc));
5995          printf("RX Bytes = %x\n", reg);
5996         reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
5997          printf("Port RX Bytes = %x\n", reg);
5998         reg = rd32(hw, I40E_GLV_RDPC(0xc));
5999          printf("RX discard = %x\n", reg);
6000         reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
6001          printf("Port RX discard = %x\n", reg);
6002
6003         reg = rd32(hw, I40E_GLV_TEPC(0xc));
6004          printf("TX errors = %x\n", reg);
6005         reg = rd32(hw, I40E_GLV_GOTCL(0xc));
6006          printf("TX Bytes = %x\n", reg);
6007
6008         reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
6009          printf("RX undersize = %x\n", reg);
6010         reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
6011          printf("RX fragments = %x\n", reg);
6012         reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
6013          printf("RX oversize = %x\n", reg);
6014         reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
6015          printf("RX length error = %x\n", reg);
6016         reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
6017          printf("mac remote fault = %x\n", reg);
6018         reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
6019          printf("mac local fault = %x\n", reg);
6020 }
6021
6022 #endif /* IXL_DEBUG_SYSCTL */
6023
6024 #ifdef PCI_IOV
6025 static int
6026 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6027 {
6028         struct i40e_hw *hw;
6029         struct ixl_vsi *vsi;
6030         struct i40e_vsi_context vsi_ctx;
6031         int i;
6032         uint16_t first_queue;
6033         enum i40e_status_code code;
6034
6035         hw = &pf->hw;
6036         vsi = &pf->vsi;
6037
6038         vsi_ctx.pf_num = hw->pf_id;
6039         vsi_ctx.uplink_seid = pf->veb_seid;
6040         vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
6041         vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6042         vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
6043
6044         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6045
6046         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6047         vsi_ctx.info.switch_id = htole16(0);
6048
6049         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
6050         vsi_ctx.info.sec_flags = 0;
6051         if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
6052                 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
6053
6054         /* TODO: If a port VLAN is set, then this needs to be changed */
6055         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6056         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6057             I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
6058
6059         vsi_ctx.info.valid_sections |=
6060             htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
6061         vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
6062         first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
6063         for (i = 0; i < IXLV_MAX_QUEUES; i++)
6064                 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
6065         for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
6066                 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
6067
6068         vsi_ctx.info.tc_mapping[0] = htole16(
6069             (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6070             (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
6071
6072         code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
6073         if (code != I40E_SUCCESS)
6074                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6075         vf->vsi.seid = vsi_ctx.seid;
6076         vf->vsi.vsi_num = vsi_ctx.vsi_number;
6077         vf->vsi.first_queue = first_queue;
6078         vf->vsi.num_queues = IXLV_MAX_QUEUES;
6079
6080         code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
6081         if (code != I40E_SUCCESS)
6082                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6083
6084         code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
6085         if (code != I40E_SUCCESS) {
6086                 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
6087                     ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6088                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6089         }
6090
6091         memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
6092         return (0);
6093 }
6094
6095 static int
6096 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6097 {
6098         struct i40e_hw *hw;
6099         int error;
6100
6101         hw = &pf->hw;
6102
6103         error = ixl_vf_alloc_vsi(pf, vf);
6104         if (error != 0)
6105                 return (error);
6106
6107         vf->vsi.hw_filters_add = 0;
6108         vf->vsi.hw_filters_del = 0;
6109         ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
6110         ixl_reconfigure_filters(&vf->vsi);
6111
6112         return (0);
6113 }
6114
6115 static void
6116 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
6117     uint32_t val)
6118 {
6119         uint32_t qtable;
6120         int index, shift;
6121
6122         /*
6123          * Two queues are mapped in a single register, so we have to do some
6124          * gymnastics to convert the queue number into a register index and
6125          * shift.
6126          */
6127         index = qnum / 2;
6128         shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
6129
6130         qtable = i40e_read_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
6131         qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
6132         qtable |= val << shift;
6133         i40e_write_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
6134 }
6135
6136 static void
6137 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
6138 {
6139         struct i40e_hw *hw;
6140         uint32_t qtable;
6141         int i;
6142
6143         hw = &pf->hw;
6144
6145         /*
6146          * Contiguous mappings aren't actually supported by the hardware,
6147          * so we have to use non-contiguous mappings.
6148          */
6149         i40e_write_rx_ctl(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
6150              I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
6151
6152         wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
6153             I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
6154
6155         for (i = 0; i < vf->vsi.num_queues; i++) {
6156                 qtable = (vf->vsi.first_queue + i) <<
6157                     I40E_VPLAN_QTABLE_QINDEX_SHIFT;
6158
6159                 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
6160         }
6161
6162         /* Map queues allocated to VF to its VSI. */
6163         for (i = 0; i < vf->vsi.num_queues; i++)
6164                 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
6165
6166         /* Set rest of VSI queues as unused. */
6167         for (; i < IXL_MAX_VSI_QUEUES; i++)
6168                 ixl_vf_map_vsi_queue(hw, vf, i,
6169                     I40E_VSILAN_QTABLE_QINDEX_0_MASK);
6170
6171         ixl_flush(hw);
6172 }
6173
6174 static void
6175 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
6176 {
6177         struct i40e_hw *hw;
6178
6179         hw = &pf->hw;
6180
6181         if (vsi->seid == 0)
6182                 return;
6183
6184         i40e_aq_delete_element(hw, vsi->seid, NULL);
6185 }
6186
6187 static void
6188 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
6189 {
6190
6191         wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
6192         ixl_flush(hw);
6193 }
6194
6195 static void
6196 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
6197 {
6198
6199         wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
6200             I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
6201         ixl_flush(hw);
6202 }
6203
6204 static void
6205 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
6206 {
6207         struct i40e_hw *hw;
6208         uint32_t vfint_reg, vpint_reg;
6209         int i;
6210
6211         hw = &pf->hw;
6212
6213         ixl_vf_vsi_release(pf, &vf->vsi);
6214
6215         /* Index 0 has a special register. */
6216         ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
6217
6218         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6219                 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
6220                 ixl_vf_disable_queue_intr(hw, vfint_reg);
6221         }
6222
6223         /* Index 0 has a special register. */
6224         ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
6225
6226         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6227                 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
6228                 ixl_vf_unregister_intr(hw, vpint_reg);
6229         }
6230
6231         vf->vsi.num_queues = 0;
6232 }
6233
6234 static int
6235 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
6236 {
6237         struct i40e_hw *hw;
6238         int i;
6239         uint16_t global_vf_num;
6240         uint32_t ciad;
6241
6242         hw = &pf->hw;
6243         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6244
6245         wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
6246              (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
6247         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6248                 ciad = rd32(hw, I40E_PF_PCI_CIAD);
6249                 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
6250                         return (0);
6251                 DELAY(1);
6252         }
6253
6254         return (ETIMEDOUT);
6255 }
6256
6257 static void
6258 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6259 {
6260         struct i40e_hw *hw;
6261         uint32_t vfrtrig;
6262
6263         hw = &pf->hw;
6264
6265         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6266         vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6267         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6268         ixl_flush(hw);
6269
6270         ixl_reinit_vf(pf, vf);
6271 }
6272
6273 static void
6274 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6275 {
6276         struct i40e_hw *hw;
6277         uint32_t vfrstat, vfrtrig;
6278         int i, error;
6279
6280         hw = &pf->hw;
6281
6282         error = ixl_flush_pcie(pf, vf);
6283         if (error != 0)
6284                 device_printf(pf->dev,
6285                     "Timed out waiting for PCIe activity to stop on VF-%d\n",
6286                     vf->vf_num);
6287
6288         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6289                 DELAY(10);
6290
6291                 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
6292                 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
6293                         break;
6294         }
6295
6296         if (i == IXL_VF_RESET_TIMEOUT)
6297                 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
6298
6299         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
6300
6301         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6302         vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6303         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6304
6305         if (vf->vsi.seid != 0)
6306                 ixl_disable_rings(&vf->vsi);
6307
6308         ixl_vf_release_resources(pf, vf);
6309         ixl_vf_setup_vsi(pf, vf);
6310         ixl_vf_map_queues(pf, vf);
6311
6312         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
6313         ixl_flush(hw);
6314 }
6315
6316 static const char *
6317 ixl_vc_opcode_str(uint16_t op)
6318 {
6319
6320         switch (op) {
6321         case I40E_VIRTCHNL_OP_VERSION:
6322                 return ("VERSION");
6323         case I40E_VIRTCHNL_OP_RESET_VF:
6324                 return ("RESET_VF");
6325         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6326                 return ("GET_VF_RESOURCES");
6327         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6328                 return ("CONFIG_TX_QUEUE");
6329         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6330                 return ("CONFIG_RX_QUEUE");
6331         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6332                 return ("CONFIG_VSI_QUEUES");
6333         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6334                 return ("CONFIG_IRQ_MAP");
6335         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6336                 return ("ENABLE_QUEUES");
6337         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6338                 return ("DISABLE_QUEUES");
6339         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6340                 return ("ADD_ETHER_ADDRESS");
6341         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6342                 return ("DEL_ETHER_ADDRESS");
6343         case I40E_VIRTCHNL_OP_ADD_VLAN:
6344                 return ("ADD_VLAN");
6345         case I40E_VIRTCHNL_OP_DEL_VLAN:
6346                 return ("DEL_VLAN");
6347         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6348                 return ("CONFIG_PROMISCUOUS_MODE");
6349         case I40E_VIRTCHNL_OP_GET_STATS:
6350                 return ("GET_STATS");
6351         case I40E_VIRTCHNL_OP_FCOE:
6352                 return ("FCOE");
6353         case I40E_VIRTCHNL_OP_EVENT:
6354                 return ("EVENT");
6355         default:
6356                 return ("UNKNOWN");
6357         }
6358 }
6359
6360 static int
6361 ixl_vc_opcode_level(uint16_t opcode)
6362 {
6363         switch (opcode) {
6364         case I40E_VIRTCHNL_OP_GET_STATS:
6365                 return (10);
6366         default:
6367                 return (5);
6368         }
6369 }
6370
6371 static void
6372 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6373     enum i40e_status_code status, void *msg, uint16_t len)
6374 {
6375         struct i40e_hw *hw;
6376         int global_vf_id;
6377
6378         hw = &pf->hw;
6379         global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
6380
6381         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
6382             "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
6383             ixl_vc_opcode_str(op), op, status, vf->vf_num);
6384
6385         i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
6386 }
6387
6388 static void
6389 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
6390 {
6391
6392         ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
6393 }
6394
6395 static void
6396 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6397     enum i40e_status_code status, const char *file, int line)
6398 {
6399
6400         I40E_VC_DEBUG(pf, 1,
6401             "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
6402             ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
6403         ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
6404 }
6405
6406 static void
6407 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6408     uint16_t msg_size)
6409 {
6410         struct i40e_virtchnl_version_info reply;
6411
6412         if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
6413                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
6414                     I40E_ERR_PARAM);
6415                 return;
6416         }
6417
6418         vf->version = ((struct i40e_virtchnl_version_info *)msg)->minor;
6419
6420         reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
6421         reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
6422         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
6423             sizeof(reply));
6424 }
6425
6426 static void
6427 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6428     uint16_t msg_size)
6429 {
6430
6431         if (msg_size != 0) {
6432                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
6433                     I40E_ERR_PARAM);
6434                 return;
6435         }
6436
6437         ixl_reset_vf(pf, vf);
6438
6439         /* No response to a reset message. */
6440 }
6441
6442 static void
6443 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6444     uint16_t msg_size)
6445 {
6446         struct i40e_virtchnl_vf_resource reply;
6447
6448         if ((vf->version == 0 && msg_size != 0) ||
6449             (vf->version == 1 && msg_size != 4)) {
6450                 device_printf(pf->dev, "Invalid GET_VF_RESOURCES message size,"
6451                     " for VF version %d.%d\n", I40E_VIRTCHNL_VERSION_MAJOR,
6452                     vf->version);
6453                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6454                     I40E_ERR_PARAM);
6455                 return;
6456         }
6457
6458         bzero(&reply, sizeof(reply));
6459
6460         if (vf->version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS)
6461                 reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
6462                                          I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
6463                                          I40E_VIRTCHNL_VF_OFFLOAD_VLAN;
6464         else
6465                 reply.vf_offload_flags = *(u32 *)msg;
6466
6467         reply.num_vsis = 1;
6468         reply.num_queue_pairs = vf->vsi.num_queues;
6469         reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
6470         reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
6471         reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
6472         reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
6473         memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
6474
6475         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6476             I40E_SUCCESS, &reply, sizeof(reply));
6477 }
6478
6479 static int
6480 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6481     struct i40e_virtchnl_txq_info *info)
6482 {
6483         struct i40e_hw *hw;
6484         struct i40e_hmc_obj_txq txq;
6485         uint16_t global_queue_num, global_vf_num;
6486         enum i40e_status_code status;
6487         uint32_t qtx_ctl;
6488
6489         hw = &pf->hw;
6490         global_queue_num = vf->vsi.first_queue + info->queue_id;
6491         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6492         bzero(&txq, sizeof(txq));
6493
6494         status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
6495         if (status != I40E_SUCCESS)
6496                 return (EINVAL);
6497
6498         txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
6499
6500         txq.head_wb_ena = info->headwb_enabled;
6501         txq.head_wb_addr = info->dma_headwb_addr;
6502         txq.qlen = info->ring_len;
6503         txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
6504         txq.rdylist_act = 0;
6505
6506         status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
6507         if (status != I40E_SUCCESS)
6508                 return (EINVAL);
6509
6510         qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
6511             (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
6512             (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
6513         wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
6514         ixl_flush(hw);
6515
6516         return (0);
6517 }
6518
6519 static int
6520 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6521     struct i40e_virtchnl_rxq_info *info)
6522 {
6523         struct i40e_hw *hw;
6524         struct i40e_hmc_obj_rxq rxq;
6525         uint16_t global_queue_num;
6526         enum i40e_status_code status;
6527
6528         hw = &pf->hw;
6529         global_queue_num = vf->vsi.first_queue + info->queue_id;
6530         bzero(&rxq, sizeof(rxq));
6531
6532         if (info->databuffer_size > IXL_VF_MAX_BUFFER)
6533                 return (EINVAL);
6534
6535         if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
6536             info->max_pkt_size < ETHER_MIN_LEN)
6537                 return (EINVAL);
6538
6539         if (info->splithdr_enabled) {
6540                 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
6541                         return (EINVAL);
6542
6543                 rxq.hsplit_0 = info->rx_split_pos &
6544                     (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
6545                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
6546                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
6547                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
6548                 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
6549
6550                 rxq.dtype = 2;
6551         }
6552
6553         status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
6554         if (status != I40E_SUCCESS)
6555                 return (EINVAL);
6556
6557         rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
6558         rxq.qlen = info->ring_len;
6559
6560         rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
6561
6562         rxq.dsize = 1;
6563         rxq.crcstrip = 1;
6564         rxq.l2tsel = 1;
6565
6566         rxq.rxmax = info->max_pkt_size;
6567         rxq.tphrdesc_ena = 1;
6568         rxq.tphwdesc_ena = 1;
6569         rxq.tphdata_ena = 1;
6570         rxq.tphhead_ena = 1;
6571         rxq.lrxqthresh = 2;
6572         rxq.prefena = 1;
6573
6574         status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
6575         if (status != I40E_SUCCESS)
6576                 return (EINVAL);
6577
6578         return (0);
6579 }
6580
6581 static void
6582 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6583     uint16_t msg_size)
6584 {
6585         struct i40e_virtchnl_vsi_queue_config_info *info;
6586         struct i40e_virtchnl_queue_pair_info *pair;
6587         int i;
6588
6589         if (msg_size < sizeof(*info)) {
6590                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6591                     I40E_ERR_PARAM);
6592                 return;
6593         }
6594
6595         info = msg;
6596         if (info->num_queue_pairs == 0) {
6597                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6598                     I40E_ERR_PARAM);
6599                 return;
6600         }
6601
6602         if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
6603                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6604                     I40E_ERR_PARAM);
6605                 return;
6606         }
6607
6608         if (info->vsi_id != vf->vsi.vsi_num) {
6609                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6610                     I40E_ERR_PARAM);
6611                 return;
6612         }
6613
6614         for (i = 0; i < info->num_queue_pairs; i++) {
6615                 pair = &info->qpair[i];
6616
6617                 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
6618                     pair->rxq.vsi_id != vf->vsi.vsi_num ||
6619                     pair->txq.queue_id != pair->rxq.queue_id ||
6620                     pair->txq.queue_id >= vf->vsi.num_queues) {
6621
6622                         i40e_send_vf_nack(pf, vf,
6623                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6624                         return;
6625                 }
6626
6627                 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
6628                         i40e_send_vf_nack(pf, vf,
6629                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6630                         return;
6631                 }
6632
6633                 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
6634                         i40e_send_vf_nack(pf, vf,
6635                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6636                         return;
6637                 }
6638         }
6639
6640         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
6641 }
6642
6643 static void
6644 ixl_vf_set_qctl(struct ixl_pf *pf,
6645     const struct i40e_virtchnl_vector_map *vector,
6646     enum i40e_queue_type cur_type, uint16_t cur_queue,
6647     enum i40e_queue_type *last_type, uint16_t *last_queue)
6648 {
6649         uint32_t offset, qctl;
6650         uint16_t itr_indx;
6651
6652         if (cur_type == I40E_QUEUE_TYPE_RX) {
6653                 offset = I40E_QINT_RQCTL(cur_queue);
6654                 itr_indx = vector->rxitr_idx;
6655         } else {
6656                 offset = I40E_QINT_TQCTL(cur_queue);
6657                 itr_indx = vector->txitr_idx;
6658         }
6659
6660         qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
6661             (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
6662             (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
6663             I40E_QINT_RQCTL_CAUSE_ENA_MASK |
6664             (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
6665
6666         wr32(&pf->hw, offset, qctl);
6667
6668         *last_type = cur_type;
6669         *last_queue = cur_queue;
6670 }
6671
6672 static void
6673 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
6674     const struct i40e_virtchnl_vector_map *vector)
6675 {
6676         struct i40e_hw *hw;
6677         u_int qindex;
6678         enum i40e_queue_type type, last_type;
6679         uint32_t lnklst_reg;
6680         uint16_t rxq_map, txq_map, cur_queue, last_queue;
6681
6682         hw = &pf->hw;
6683
6684         rxq_map = vector->rxq_map;
6685         txq_map = vector->txq_map;
6686
6687         last_queue = IXL_END_OF_INTR_LNKLST;
6688         last_type = I40E_QUEUE_TYPE_RX;
6689
6690         /*
6691          * The datasheet says to optimize performance, RX queues and TX queues
6692          * should be interleaved in the interrupt linked list, so we process
6693          * both at once here.
6694          */
6695         while ((rxq_map != 0) || (txq_map != 0)) {
6696                 if (txq_map != 0) {
6697                         qindex = ffs(txq_map) - 1;
6698                         type = I40E_QUEUE_TYPE_TX;
6699                         cur_queue = vf->vsi.first_queue + qindex;
6700                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
6701                             &last_type, &last_queue);
6702                         txq_map &= ~(1 << qindex);
6703                 }
6704
6705                 if (rxq_map != 0) {
6706                         qindex = ffs(rxq_map) - 1;
6707                         type = I40E_QUEUE_TYPE_RX;
6708                         cur_queue = vf->vsi.first_queue + qindex;
6709                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
6710                             &last_type, &last_queue);
6711                         rxq_map &= ~(1 << qindex);
6712                 }
6713         }
6714
6715         if (vector->vector_id == 0)
6716                 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
6717         else
6718                 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
6719                     vf->vf_num);
6720         wr32(hw, lnklst_reg,
6721             (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
6722             (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
6723
6724         ixl_flush(hw);
6725 }
6726
6727 static void
6728 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6729     uint16_t msg_size)
6730 {
6731         struct i40e_virtchnl_irq_map_info *map;
6732         struct i40e_virtchnl_vector_map *vector;
6733         struct i40e_hw *hw;
6734         int i, largest_txq, largest_rxq;
6735
6736         hw = &pf->hw;
6737
6738         if (msg_size < sizeof(*map)) {
6739                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6740                     I40E_ERR_PARAM);
6741                 return;
6742         }
6743
6744         map = msg;
6745         if (map->num_vectors == 0) {
6746                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6747                     I40E_ERR_PARAM);
6748                 return;
6749         }
6750
6751         if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
6752                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6753                     I40E_ERR_PARAM);
6754                 return;
6755         }
6756
6757         for (i = 0; i < map->num_vectors; i++) {
6758                 vector = &map->vecmap[i];
6759
6760                 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
6761                     vector->vsi_id != vf->vsi.vsi_num) {
6762                         i40e_send_vf_nack(pf, vf,
6763                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
6764                         return;
6765                 }
6766
6767                 if (vector->rxq_map != 0) {
6768                         largest_rxq = fls(vector->rxq_map) - 1;
6769                         if (largest_rxq >= vf->vsi.num_queues) {
6770                                 i40e_send_vf_nack(pf, vf,
6771                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6772                                     I40E_ERR_PARAM);
6773                                 return;
6774                         }
6775                 }
6776
6777                 if (vector->txq_map != 0) {
6778                         largest_txq = fls(vector->txq_map) - 1;
6779                         if (largest_txq >= vf->vsi.num_queues) {
6780                                 i40e_send_vf_nack(pf, vf,
6781                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6782                                     I40E_ERR_PARAM);
6783                                 return;
6784                         }
6785                 }
6786
6787                 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6788                     vector->txitr_idx > IXL_MAX_ITR_IDX) {
6789                         i40e_send_vf_nack(pf, vf,
6790                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6791                             I40E_ERR_PARAM);
6792                         return;
6793                 }
6794
6795                 ixl_vf_config_vector(pf, vf, vector);
6796         }
6797
6798         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6799 }
6800
6801 static void
6802 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6803     uint16_t msg_size)
6804 {
6805         struct i40e_virtchnl_queue_select *select;
6806         int error;
6807
6808         if (msg_size != sizeof(*select)) {
6809                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6810                     I40E_ERR_PARAM);
6811                 return;
6812         }
6813
6814         select = msg;
6815         if (select->vsi_id != vf->vsi.vsi_num ||
6816             select->rx_queues == 0 || select->tx_queues == 0) {
6817                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6818                     I40E_ERR_PARAM);
6819                 return;
6820         }
6821
6822         error = ixl_enable_rings(&vf->vsi);
6823         if (error) {
6824                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6825                     I40E_ERR_TIMEOUT);
6826                 return;
6827         }
6828
6829         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6830 }
6831
6832 static void
6833 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6834     void *msg, uint16_t msg_size)
6835 {
6836         struct i40e_virtchnl_queue_select *select;
6837         int error;
6838
6839         if (msg_size != sizeof(*select)) {
6840                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6841                     I40E_ERR_PARAM);
6842                 return;
6843         }
6844
6845         select = msg;
6846         if (select->vsi_id != vf->vsi.vsi_num ||
6847             select->rx_queues == 0 || select->tx_queues == 0) {
6848                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6849                     I40E_ERR_PARAM);
6850                 return;
6851         }
6852
6853         error = ixl_disable_rings(&vf->vsi);
6854         if (error) {
6855                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6856                     I40E_ERR_TIMEOUT);
6857                 return;
6858         }
6859
6860         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6861 }
6862
6863 static boolean_t
6864 ixl_zero_mac(const uint8_t *addr)
6865 {
6866         uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6867
6868         return (cmp_etheraddr(addr, zero));
6869 }
6870
6871 static boolean_t
6872 ixl_bcast_mac(const uint8_t *addr)
6873 {
6874
6875         return (cmp_etheraddr(addr, ixl_bcast_addr));
6876 }
6877
6878 static int
6879 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6880 {
6881
6882         if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6883                 return (EINVAL);
6884
6885         /*
6886          * If the VF is not allowed to change its MAC address, don't let it
6887          * set a MAC filter for an address that is not a multicast address and
6888          * is not its assigned MAC.
6889          */
6890         if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6891             !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6892                 return (EPERM);
6893
6894         return (0);
6895 }
6896
6897 static void
6898 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6899     uint16_t msg_size)
6900 {
6901         struct i40e_virtchnl_ether_addr_list *addr_list;
6902         struct i40e_virtchnl_ether_addr *addr;
6903         struct ixl_vsi *vsi;
6904         int i;
6905         size_t expected_size;
6906
6907         vsi = &vf->vsi;
6908
6909         if (msg_size < sizeof(*addr_list)) {
6910                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6911                     I40E_ERR_PARAM);
6912                 return;
6913         }
6914
6915         addr_list = msg;
6916         expected_size = sizeof(*addr_list) +
6917             addr_list->num_elements * sizeof(*addr);
6918
6919         if (addr_list->num_elements == 0 ||
6920             addr_list->vsi_id != vsi->vsi_num ||
6921             msg_size != expected_size) {
6922                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6923                     I40E_ERR_PARAM);
6924                 return;
6925         }
6926
6927         for (i = 0; i < addr_list->num_elements; i++) {
6928                 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6929                         i40e_send_vf_nack(pf, vf,
6930                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6931                         return;
6932                 }
6933         }
6934
6935         for (i = 0; i < addr_list->num_elements; i++) {
6936                 addr = &addr_list->list[i];
6937                 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6938         }
6939
6940         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6941 }
6942
6943 static void
6944 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6945     uint16_t msg_size)
6946 {
6947         struct i40e_virtchnl_ether_addr_list *addr_list;
6948         struct i40e_virtchnl_ether_addr *addr;
6949         size_t expected_size;
6950         int i;
6951
6952         if (msg_size < sizeof(*addr_list)) {
6953                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6954                     I40E_ERR_PARAM);
6955                 return;
6956         }
6957
6958         addr_list = msg;
6959         expected_size = sizeof(*addr_list) +
6960             addr_list->num_elements * sizeof(*addr);
6961
6962         if (addr_list->num_elements == 0 ||
6963             addr_list->vsi_id != vf->vsi.vsi_num ||
6964             msg_size != expected_size) {
6965                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6966                     I40E_ERR_PARAM);
6967                 return;
6968         }
6969
6970         for (i = 0; i < addr_list->num_elements; i++) {
6971                 addr = &addr_list->list[i];
6972                 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6973                         i40e_send_vf_nack(pf, vf,
6974                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6975                         return;
6976                 }
6977         }
6978
6979         for (i = 0; i < addr_list->num_elements; i++) {
6980                 addr = &addr_list->list[i];
6981                 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6982         }
6983
6984         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6985 }
6986
6987 static enum i40e_status_code
6988 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6989 {
6990         struct i40e_vsi_context vsi_ctx;
6991
6992         vsi_ctx.seid = vf->vsi.seid;
6993
6994         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6995         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6996         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6997             I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6998         return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6999 }
7000
7001 static void
7002 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7003     uint16_t msg_size)
7004 {
7005         struct i40e_virtchnl_vlan_filter_list *filter_list;
7006         enum i40e_status_code code;
7007         size_t expected_size;
7008         int i;
7009
7010         if (msg_size < sizeof(*filter_list)) {
7011                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7012                     I40E_ERR_PARAM);
7013                 return;
7014         }
7015
7016         filter_list = msg;
7017         expected_size = sizeof(*filter_list) +
7018             filter_list->num_elements * sizeof(uint16_t);
7019         if (filter_list->num_elements == 0 ||
7020             filter_list->vsi_id != vf->vsi.vsi_num ||
7021             msg_size != expected_size) {
7022                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7023                     I40E_ERR_PARAM);
7024                 return;
7025         }
7026
7027         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7028                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7029                     I40E_ERR_PARAM);
7030                 return;
7031         }
7032
7033         for (i = 0; i < filter_list->num_elements; i++) {
7034                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7035                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7036                             I40E_ERR_PARAM);
7037                         return;
7038                 }
7039         }
7040
7041         code = ixl_vf_enable_vlan_strip(pf, vf);
7042         if (code != I40E_SUCCESS) {
7043                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7044                     I40E_ERR_PARAM);
7045         }
7046
7047         for (i = 0; i < filter_list->num_elements; i++)
7048                 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7049
7050         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
7051 }
7052
7053 static void
7054 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7055     uint16_t msg_size)
7056 {
7057         struct i40e_virtchnl_vlan_filter_list *filter_list;
7058         int i;
7059         size_t expected_size;
7060
7061         if (msg_size < sizeof(*filter_list)) {
7062                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7063                     I40E_ERR_PARAM);
7064                 return;
7065         }
7066
7067         filter_list = msg;
7068         expected_size = sizeof(*filter_list) +
7069             filter_list->num_elements * sizeof(uint16_t);
7070         if (filter_list->num_elements == 0 ||
7071             filter_list->vsi_id != vf->vsi.vsi_num ||
7072             msg_size != expected_size) {
7073                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7074                     I40E_ERR_PARAM);
7075                 return;
7076         }
7077
7078         for (i = 0; i < filter_list->num_elements; i++) {
7079                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7080                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7081                             I40E_ERR_PARAM);
7082                         return;
7083                 }
7084         }
7085
7086         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7087                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7088                     I40E_ERR_PARAM);
7089                 return;
7090         }
7091
7092         for (i = 0; i < filter_list->num_elements; i++)
7093                 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7094
7095         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
7096 }
7097
7098 static void
7099 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
7100     void *msg, uint16_t msg_size)
7101 {
7102         struct i40e_virtchnl_promisc_info *info;
7103         enum i40e_status_code code;
7104
7105         if (msg_size != sizeof(*info)) {
7106                 i40e_send_vf_nack(pf, vf,
7107                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7108                 return;
7109         }
7110
7111         if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) {
7112                 i40e_send_vf_nack(pf, vf,
7113                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7114                 return;
7115         }
7116
7117         info = msg;
7118         if (info->vsi_id != vf->vsi.vsi_num) {
7119                 i40e_send_vf_nack(pf, vf,
7120                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7121                 return;
7122         }
7123
7124         code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
7125             info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
7126         if (code != I40E_SUCCESS) {
7127                 i40e_send_vf_nack(pf, vf,
7128                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7129                 return;
7130         }
7131
7132         code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
7133             info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
7134         if (code != I40E_SUCCESS) {
7135                 i40e_send_vf_nack(pf, vf,
7136                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7137                 return;
7138         }
7139
7140         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
7141 }
7142
7143 static void
7144 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7145     uint16_t msg_size)
7146 {
7147         struct i40e_virtchnl_queue_select *queue;
7148
7149         if (msg_size != sizeof(*queue)) {
7150                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7151                     I40E_ERR_PARAM);
7152                 return;
7153         }
7154
7155         queue = msg;
7156         if (queue->vsi_id != vf->vsi.vsi_num) {
7157                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7158                     I40E_ERR_PARAM);
7159                 return;
7160         }
7161
7162         ixl_update_eth_stats(&vf->vsi);
7163
7164         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7165             I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
7166 }
7167
7168 static void
7169 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
7170 {
7171         struct ixl_vf *vf;
7172         void *msg;
7173         uint16_t vf_num, msg_size;
7174         uint32_t opcode;
7175
7176         vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
7177         opcode = le32toh(event->desc.cookie_high);
7178
7179         if (vf_num >= pf->num_vfs) {
7180                 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
7181                 return;
7182         }
7183
7184         vf = &pf->vfs[vf_num];
7185         msg = event->msg_buf;
7186         msg_size = event->msg_len;
7187
7188         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
7189             "Got msg %s(%d) from VF-%d of size %d\n",
7190             ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
7191
7192         switch (opcode) {
7193         case I40E_VIRTCHNL_OP_VERSION:
7194                 ixl_vf_version_msg(pf, vf, msg, msg_size);
7195                 break;
7196         case I40E_VIRTCHNL_OP_RESET_VF:
7197                 ixl_vf_reset_msg(pf, vf, msg, msg_size);
7198                 break;
7199         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
7200                 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
7201                 break;
7202         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
7203                 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
7204                 break;
7205         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
7206                 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
7207                 break;
7208         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
7209                 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
7210                 break;
7211         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
7212                 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
7213                 break;
7214         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
7215                 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
7216                 break;
7217         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
7218                 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
7219                 break;
7220         case I40E_VIRTCHNL_OP_ADD_VLAN:
7221                 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
7222                 break;
7223         case I40E_VIRTCHNL_OP_DEL_VLAN:
7224                 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
7225                 break;
7226         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
7227                 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
7228                 break;
7229         case I40E_VIRTCHNL_OP_GET_STATS:
7230                 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
7231                 break;
7232
7233         /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
7234         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
7235         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
7236         default:
7237                 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
7238                 break;
7239         }
7240 }
7241
7242 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
7243 static void
7244 ixl_handle_vflr(void *arg, int pending)
7245 {
7246         struct ixl_pf *pf;
7247         struct i40e_hw *hw;
7248         uint16_t global_vf_num;
7249         uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
7250         int i;
7251
7252         pf = arg;
7253         hw = &pf->hw;
7254
7255         IXL_PF_LOCK(pf);
7256         for (i = 0; i < pf->num_vfs; i++) {
7257                 global_vf_num = hw->func_caps.vf_base_id + i;
7258
7259                 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
7260                 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
7261                 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
7262                 if (vflrstat & vflrstat_mask) {
7263                         wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
7264                             vflrstat_mask);
7265
7266                         ixl_reinit_vf(pf, &pf->vfs[i]);
7267                 }
7268         }
7269
7270         icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
7271         icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
7272         wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
7273         ixl_flush(hw);
7274
7275         IXL_PF_UNLOCK(pf);
7276 }
7277
7278 static int
7279 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
7280 {
7281
7282         switch (err) {
7283         case I40E_AQ_RC_EPERM:
7284                 return (EPERM);
7285         case I40E_AQ_RC_ENOENT:
7286                 return (ENOENT);
7287         case I40E_AQ_RC_ESRCH:
7288                 return (ESRCH);
7289         case I40E_AQ_RC_EINTR:
7290                 return (EINTR);
7291         case I40E_AQ_RC_EIO:
7292                 return (EIO);
7293         case I40E_AQ_RC_ENXIO:
7294                 return (ENXIO);
7295         case I40E_AQ_RC_E2BIG:
7296                 return (E2BIG);
7297         case I40E_AQ_RC_EAGAIN:
7298                 return (EAGAIN);
7299         case I40E_AQ_RC_ENOMEM:
7300                 return (ENOMEM);
7301         case I40E_AQ_RC_EACCES:
7302                 return (EACCES);
7303         case I40E_AQ_RC_EFAULT:
7304                 return (EFAULT);
7305         case I40E_AQ_RC_EBUSY:
7306                 return (EBUSY);
7307         case I40E_AQ_RC_EEXIST:
7308                 return (EEXIST);
7309         case I40E_AQ_RC_EINVAL:
7310                 return (EINVAL);
7311         case I40E_AQ_RC_ENOTTY:
7312                 return (ENOTTY);
7313         case I40E_AQ_RC_ENOSPC:
7314                 return (ENOSPC);
7315         case I40E_AQ_RC_ENOSYS:
7316                 return (ENOSYS);
7317         case I40E_AQ_RC_ERANGE:
7318                 return (ERANGE);
7319         case I40E_AQ_RC_EFLUSHED:
7320                 return (EINVAL);        /* No exact equivalent in errno.h */
7321         case I40E_AQ_RC_BAD_ADDR:
7322                 return (EFAULT);
7323         case I40E_AQ_RC_EMODE:
7324                 return (EPERM);
7325         case I40E_AQ_RC_EFBIG:
7326                 return (EFBIG);
7327         default:
7328                 return (EINVAL);
7329         }
7330 }
7331
7332 static int
7333 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
7334 {
7335         struct ixl_pf *pf;
7336         struct i40e_hw *hw;
7337         struct ixl_vsi *pf_vsi;
7338         enum i40e_status_code ret;
7339         int i, error;
7340
7341         pf = device_get_softc(dev);
7342         hw = &pf->hw;
7343         pf_vsi = &pf->vsi;
7344
7345         IXL_PF_LOCK(pf);
7346         pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
7347             M_ZERO);
7348
7349         if (pf->vfs == NULL) {
7350                 error = ENOMEM;
7351                 goto fail;
7352         }
7353
7354         for (i = 0; i < num_vfs; i++)
7355                 sysctl_ctx_init(&pf->vfs[i].ctx);
7356
7357         ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
7358             1, FALSE, &pf->veb_seid, FALSE, NULL);
7359         if (ret != I40E_SUCCESS) {
7360                 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
7361                 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
7362                     error);
7363                 goto fail;
7364         }
7365
7366         // TODO: [Configure MSI-X here]
7367         ixl_enable_adminq(hw);
7368
7369         pf->num_vfs = num_vfs;
7370         IXL_PF_UNLOCK(pf);
7371         return (0);
7372
7373 fail:
7374         free(pf->vfs, M_IXL);
7375         pf->vfs = NULL;
7376         IXL_PF_UNLOCK(pf);
7377         return (error);
7378 }
7379
7380 static void
7381 ixl_iov_uninit(device_t dev)
7382 {
7383         struct ixl_pf *pf;
7384         struct i40e_hw *hw;
7385         struct ixl_vsi *vsi;
7386         struct ifnet *ifp;
7387         struct ixl_vf *vfs;
7388         int i, num_vfs;
7389
7390         pf = device_get_softc(dev);
7391         hw = &pf->hw;
7392         vsi = &pf->vsi;
7393         ifp = vsi->ifp;
7394
7395         IXL_PF_LOCK(pf);
7396         for (i = 0; i < pf->num_vfs; i++) {
7397                 if (pf->vfs[i].vsi.seid != 0)
7398                         i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
7399         }
7400
7401         if (pf->veb_seid != 0) {
7402                 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
7403                 pf->veb_seid = 0;
7404         }
7405
7406         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
7407                 ixl_disable_intr(vsi);
7408
7409         vfs = pf->vfs;
7410         num_vfs = pf->num_vfs;
7411
7412         pf->vfs = NULL;
7413         pf->num_vfs = 0;
7414         IXL_PF_UNLOCK(pf);
7415
7416         /* Do this after the unlock as sysctl_ctx_free might sleep. */
7417         for (i = 0; i < num_vfs; i++)
7418                 sysctl_ctx_free(&vfs[i].ctx);
7419         free(vfs, M_IXL);
7420 }
7421
7422 static int
7423 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
7424 {
7425         char sysctl_name[QUEUE_NAME_LEN];
7426         struct ixl_pf *pf;
7427         struct ixl_vf *vf;
7428         const void *mac;
7429         size_t size;
7430         int error;
7431
7432         pf = device_get_softc(dev);
7433         vf = &pf->vfs[vfnum];
7434
7435         IXL_PF_LOCK(pf);
7436         vf->vf_num = vfnum;
7437
7438         vf->vsi.back = pf;
7439         vf->vf_flags = VF_FLAG_ENABLED;
7440         SLIST_INIT(&vf->vsi.ftl);
7441
7442         error = ixl_vf_setup_vsi(pf, vf);
7443         if (error != 0)
7444                 goto out;
7445
7446         if (nvlist_exists_binary(params, "mac-addr")) {
7447                 mac = nvlist_get_binary(params, "mac-addr", &size);
7448                 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
7449
7450                 if (nvlist_get_bool(params, "allow-set-mac"))
7451                         vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7452         } else
7453                 /*
7454                  * If the administrator has not specified a MAC address then
7455                  * we must allow the VF to choose one.
7456                  */
7457                 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7458
7459         if (nvlist_get_bool(params, "mac-anti-spoof"))
7460                 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
7461
7462         if (nvlist_get_bool(params, "allow-promisc"))
7463                 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
7464
7465         /* TODO: Get VLAN that PF has set for the VF */
7466
7467         vf->vf_flags |= VF_FLAG_VLAN_CAP;
7468
7469         ixl_reset_vf(pf, vf);
7470 out:
7471         IXL_PF_UNLOCK(pf);
7472         if (error == 0) {
7473                 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
7474                 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
7475         }
7476
7477         return (error);
7478 }
7479 #endif /* PCI_IOV */