sys/dev/ixl/if_ixl.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifndef IXL_STANDALONE_BUILD
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "opt_rss.h"
  39 #endif
  40
  41 #include "ixl.h"
  42 #include "ixl_pf.h"
  43
  44 #ifdef RSS
  45 #include <net/rss_config.h>
  46 #endif
  47
  48 /*********************************************************************
  49  *  Driver version
  50  *********************************************************************/
  51 char ixl_driver_version[] = "1.4.27-k";
  52
  53 /*********************************************************************
  54  *  PCI Device ID Table
  55  *
  56  *  Used by probe to select devices to load on
  57  *  Last field stores an index into ixl_strings
  58  *  Last entry must be all 0s
  59  *
  60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  61  *********************************************************************/
  62
  63 static ixl_vendor_info_t ixl_vendor_info_array[] =
  64 {
  65         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
  66         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
  67         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
  68         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
  69         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
  70         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
  71         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
  72         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
  73         /* required last entry */
  74         {0, 0, 0, 0, 0}
  75 };
  76
  77 /*********************************************************************
  78  *  Table of branding strings
  79  *********************************************************************/
  80
  81 static char    *ixl_strings[] = {
  82         "Intel(R) Ethernet Connection XL710 Driver"
  83 };
  84
  85
  86 /*********************************************************************
  87  *  Function prototypes
  88  *********************************************************************/
  89 static int      ixl_probe(device_t);
  90 static int      ixl_attach(device_t);
  91 static int      ixl_detach(device_t);
  92 static int      ixl_shutdown(device_t);
  93 static int      ixl_get_hw_capabilities(struct ixl_pf *);
  94 static void     ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
  95 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
  96 static void     ixl_init(void *);
  97 static void     ixl_init_locked(struct ixl_pf *);
  98 static void     ixl_stop(struct ixl_pf *);
  99 static void     ixl_stop_locked(struct ixl_pf *);
 100 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
 101 static int      ixl_media_change(struct ifnet *);
 102 static void     ixl_update_link_status(struct ixl_pf *);
 103 static int      ixl_allocate_pci_resources(struct ixl_pf *);
 104 static u16      ixl_get_bus_info(struct i40e_hw *, device_t);
 105 static int      ixl_setup_stations(struct ixl_pf *);
 106 static int      ixl_switch_config(struct ixl_pf *);
 107 static int      ixl_initialize_vsi(struct ixl_vsi *);
 108
 109 static int      ixl_setup_adminq_msix(struct ixl_pf *);
 110 static int      ixl_setup_adminq_tq(struct ixl_pf *);
 111 static int      ixl_setup_queue_msix(struct ixl_vsi *);
 112 static int      ixl_setup_queue_tqs(struct ixl_vsi *);
 113 static int      ixl_teardown_adminq_msix(struct ixl_pf *);
 114 static int      ixl_teardown_queue_msix(struct ixl_vsi *);
 115 static void     ixl_configure_intr0_msix(struct ixl_pf *);
 116 static void     ixl_configure_queue_intr_msix(struct ixl_pf *);
 117 static void     ixl_free_queue_tqs(struct ixl_vsi *);
 118 static void     ixl_free_adminq_tq(struct ixl_pf *);
 119
 120 static int      ixl_assign_vsi_legacy(struct ixl_pf *);
 121 static int      ixl_init_msix(struct ixl_pf *);
 122 static void     ixl_configure_itr(struct ixl_pf *);
 123 static void     ixl_configure_legacy(struct ixl_pf *);
 124 static void     ixl_free_pci_resources(struct ixl_pf *);
 125 static void     ixl_local_timer(void *);
 126 static int      ixl_setup_interface(device_t, struct ixl_vsi *);
 127 static void     ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
 128 static void     ixl_config_rss(struct ixl_vsi *);
 129 static void     ixl_set_queue_rx_itr(struct ixl_queue *);
 130 static void     ixl_set_queue_tx_itr(struct ixl_queue *);
 131 static int      ixl_set_advertised_speeds(struct ixl_pf *, int);
 132 static void     ixl_get_initial_advertised_speeds(struct ixl_pf *);
 133
 134 static int      ixl_enable_rings(struct ixl_vsi *);
 135 static int      ixl_disable_rings(struct ixl_vsi *);
 136 static void     ixl_enable_intr(struct ixl_vsi *);
 137 static void     ixl_disable_intr(struct ixl_vsi *);
 138 static void     ixl_disable_rings_intr(struct ixl_vsi *);
 139
 140 static void     ixl_enable_adminq(struct i40e_hw *);
 141 static void     ixl_disable_adminq(struct i40e_hw *);
 142 static void     ixl_enable_queue(struct i40e_hw *, int);
 143 static void     ixl_disable_queue(struct i40e_hw *, int);
 144 static void     ixl_enable_legacy(struct i40e_hw *);
 145 static void     ixl_disable_legacy(struct i40e_hw *);
 146
 147 static void     ixl_set_promisc(struct ixl_vsi *);
 148 static void     ixl_add_multi(struct ixl_vsi *);
 149 static void     ixl_del_multi(struct ixl_vsi *);
 150 static void     ixl_register_vlan(void *, struct ifnet *, u16);
 151 static void     ixl_unregister_vlan(void *, struct ifnet *, u16);
 152 static void     ixl_setup_vlan_filters(struct ixl_vsi *);
 153
 154 static void     ixl_init_filters(struct ixl_vsi *);
 155 static void     ixl_reconfigure_filters(struct ixl_vsi *vsi);
 156 static void     ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
 157 static void     ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
 158 static void     ixl_add_hw_filters(struct ixl_vsi *, int, int);
 159 static void     ixl_del_hw_filters(struct ixl_vsi *, int);
 160 static struct ixl_mac_filter *
 161                 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
 162 static void     ixl_add_mc_filter(struct ixl_vsi *, u8 *);
 163 static void     ixl_free_mac_filters(struct ixl_vsi *vsi);
 164
 165 /* Sysctls*/
 166 static void     ixl_add_device_sysctls(struct ixl_pf *);
 167
 168 static int      ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
 169 static int      ixl_set_advertise(SYSCTL_HANDLER_ARGS);
 170 static int      ixl_current_speed(SYSCTL_HANDLER_ARGS);
 171 static int      ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
 172
 173 #ifdef IXL_DEBUG_SYSCTL
 174 static int      ixl_debug_info(SYSCTL_HANDLER_ARGS);
 175 static void     ixl_print_debug_info(struct ixl_pf *);
 176
 177 static int      ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
 178 static int      ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
 179 static int      ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 180 static int      ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
 181 static int      ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
 182 #endif
 183
 184 /* The MSI/X Interrupt handlers */
 185 static void     ixl_intr(void *);
 186 static void     ixl_msix_que(void *);
 187 static void     ixl_msix_adminq(void *);
 188 static void     ixl_handle_mdd_event(struct ixl_pf *);
 189
 190 /* Deferred interrupt tasklets */
 191 static void     ixl_do_adminq(void *, int);
 192
 193 /* Statistics */
 194 static void     ixl_add_hw_stats(struct ixl_pf *);
 195 static void     ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
 196                     struct sysctl_oid_list *, struct i40e_hw_port_stats *);
 197 static void     ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
 198                     struct sysctl_oid_list *,
 199                     struct i40e_eth_stats *);
 200 static void     ixl_update_stats_counters(struct ixl_pf *);
 201 static void     ixl_update_eth_stats(struct ixl_vsi *);
 202 static void     ixl_update_vsi_stats(struct ixl_vsi *);
 203 static void     ixl_pf_reset_stats(struct ixl_pf *);
 204 static void     ixl_vsi_reset_stats(struct ixl_vsi *);
 205 static void     ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
 206                     u64 *, u64 *);
 207 static void     ixl_stat_update32(struct i40e_hw *, u32, bool,
 208                     u64 *, u64 *);
 209 /* NVM update */
 210 static int      ixl_handle_nvmupd_cmd(struct ixl_pf *, struct ifdrv *);
 211 static void     ixl_handle_empr_reset(struct ixl_pf *);
 212 static int      ixl_rebuild_hw_structs_after_reset(struct ixl_pf *);
 213
 214 /* Debug helper functions */
 215 #ifdef IXL_DEBUG
 216 static void     ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *);
 217 #endif
 218
 219 #ifdef PCI_IOV
 220 static int      ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
 221
 222 static int      ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
 223 static void     ixl_iov_uninit(device_t dev);
 224 static int      ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
 225
 226 static void     ixl_handle_vf_msg(struct ixl_pf *,
 227                     struct i40e_arq_event_info *);
 228 static void     ixl_handle_vflr(void *arg, int pending);
 229
 230 static void     ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 231 static void     ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 232 #endif
 233
 234 /*********************************************************************
 235  *  FreeBSD Device Interface Entry Points
 236  *********************************************************************/
 237
 238 static device_method_t ixl_methods[] = {
 239         /* Device interface */
 240         DEVMETHOD(device_probe, ixl_probe),
 241         DEVMETHOD(device_attach, ixl_attach),
 242         DEVMETHOD(device_detach, ixl_detach),
 243         DEVMETHOD(device_shutdown, ixl_shutdown),
 244 #ifdef PCI_IOV
 245         DEVMETHOD(pci_iov_init, ixl_iov_init),
 246         DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
 247         DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
 248 #endif
 249         {0, 0}
 250 };
 251
 252 static driver_t ixl_driver = {
 253         "ixl", ixl_methods, sizeof(struct ixl_pf),
 254 };
 255
 256 devclass_t ixl_devclass;
 257 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
 258
 259 MODULE_DEPEND(ixl, pci, 1, 1, 1);
 260 MODULE_DEPEND(ixl, ether, 1, 1, 1);
 261 #ifdef DEV_NETMAP
 262 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
 263 #endif /* DEV_NETMAP */
 264
 265 /*
 266 ** Global reset mutex
 267 */
 268 static struct mtx ixl_reset_mtx;
 269
 270 /*
 271 ** TUNEABLE PARAMETERS:
 272 */
 273
 274 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
 275                    "IXL driver parameters");
 276
 277 /*
 278  * MSIX should be the default for best performance,
 279  * but this allows it to be forced off for testing.
 280  */
 281 static int ixl_enable_msix = 1;
 282 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
 283 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
 284     "Enable MSI-X interrupts");
 285
 286 /*
 287 ** Number of descriptors per ring:
 288 **   - TX and RX are the same size
 289 */
 290 static int ixl_ringsz = DEFAULT_RING;
 291 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
 292 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
 293     &ixl_ringsz, 0, "Descriptor Ring Size");
 294
 295 /*
 296 ** This can be set manually, if left as 0 the
 297 ** number of queues will be calculated based
 298 ** on cpus and msix vectors available.
 299 */
 300 int ixl_max_queues = 0;
 301 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
 302 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
 303     &ixl_max_queues, 0, "Number of Queues");
 304
 305 /*
 306 ** Controls for Interrupt Throttling
 307 **      - true/false for dynamic adjustment
 308 **      - default values for static ITR
 309 */
 310 int ixl_dynamic_rx_itr = 1;
 311 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
 312 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
 313     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
 314
 315 int ixl_dynamic_tx_itr = 1;
 316 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
 317 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
 318     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
 319
 320 int ixl_rx_itr = IXL_ITR_8K;
 321 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
 322 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
 323     &ixl_rx_itr, 0, "RX Interrupt Rate");
 324
 325 int ixl_tx_itr = IXL_ITR_4K;
 326 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
 327 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
 328     &ixl_tx_itr, 0, "TX Interrupt Rate");
 329
 330 #ifdef IXL_FDIR
 331 static int ixl_enable_fdir = 1;
 332 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
 333 /* Rate at which we sample */
 334 int ixl_atr_rate = 20;
 335 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
 336 #endif
 337
 338 #ifdef DEV_NETMAP
 339 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
 340 #include <dev/netmap/if_ixl_netmap.h>
 341 #endif /* DEV_NETMAP */
 342
 343 static char *ixl_fc_string[6] = {
 344         "None",
 345         "Rx",
 346         "Tx",
 347         "Full",
 348         "Priority",
 349         "Default"
 350 };
 351
 352 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
 353
 354 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
 355     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 356
 357 /*********************************************************************
 358  *  Device identification routine
 359  *
 360  *  ixl_probe determines if the driver should be loaded on
 361  *  the hardware based on PCI vendor/device id of the device.
 362  *
 363  *  return BUS_PROBE_DEFAULT on success, positive on failure
 364  *********************************************************************/
 365
 366 static int
 367 ixl_probe(device_t dev)
 368 {
 369         ixl_vendor_info_t *ent;
 370
 371         u16     pci_vendor_id, pci_device_id;
 372         u16     pci_subvendor_id, pci_subdevice_id;
 373         char    device_name[256];
 374         static bool lock_init = FALSE;
 375
 376 #if 0
 377         INIT_DEBUGOUT("ixl_probe: begin");
 378 #endif
 379         pci_vendor_id = pci_get_vendor(dev);
 380         if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
 381                 return (ENXIO);
 382
 383         pci_device_id = pci_get_device(dev);
 384         pci_subvendor_id = pci_get_subvendor(dev);
 385         pci_subdevice_id = pci_get_subdevice(dev);
 386
 387         ent = ixl_vendor_info_array;
 388         while (ent->vendor_id != 0) {
 389                 if ((pci_vendor_id == ent->vendor_id) &&
 390                     (pci_device_id == ent->device_id) &&
 391
 392                     ((pci_subvendor_id == ent->subvendor_id) ||
 393                      (ent->subvendor_id == 0)) &&
 394
 395                     ((pci_subdevice_id == ent->subdevice_id) ||
 396                      (ent->subdevice_id == 0))) {
 397                         sprintf(device_name, "%s, Version - %s",
 398                                 ixl_strings[ent->index],
 399                                 ixl_driver_version);
 400                         device_set_desc_copy(dev, device_name);
 401                         /* One shot mutex init */
 402                         if (lock_init == FALSE) {
 403                                 lock_init = TRUE;
 404                                 mtx_init(&ixl_reset_mtx,
 405                                     "ixl_reset",
 406                                     "IXL RESET Lock", MTX_DEF);
 407                         }
 408                         return (BUS_PROBE_DEFAULT);
 409                 }
 410                 ent++;
 411         }
 412         return (ENXIO);
 413 }
 414
 415 /*********************************************************************
 416  *  Device initialization routine
 417  *
 418  *  The attach entry point is called when the driver is being loaded.
 419  *  This routine identifies the type of hardware, allocates all resources
 420  *  and initializes the hardware.
 421  *
 422  *  return 0 on success, positive on failure
 423  *********************************************************************/
 424
 425 static int
 426 ixl_attach(device_t dev)
 427 {
 428         struct ixl_pf   *pf;
 429         struct i40e_hw  *hw;
 430         struct ixl_vsi  *vsi;
 431         u16             bus;
 432         int             error = 0;
 433 #ifdef PCI_IOV
 434         nvlist_t        *pf_schema, *vf_schema;
 435         int             iov_error;
 436 #endif
 437
 438         INIT_DEBUGOUT("ixl_attach: begin");
 439
 440         /* Allocate, clear, and link in our primary soft structure */
 441         pf = device_get_softc(dev);
 442         pf->dev = pf->osdep.dev = dev;
 443         hw = &pf->hw;
 444
 445         /*
 446         ** Note this assumes we have a single embedded VSI,
 447         ** this could be enhanced later to allocate multiple
 448         */
 449         vsi = &pf->vsi;
 450         vsi->dev = pf->dev;
 451
 452         /* Core Lock Init*/
 453         IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
 454
 455         /* Set up the timer callout */
 456         callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
 457
 458         /* Save off the PCI information */
 459         hw->vendor_id = pci_get_vendor(dev);
 460         hw->device_id = pci_get_device(dev);
 461         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
 462         hw->subsystem_vendor_id =
 463             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 464         hw->subsystem_device_id =
 465             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 466
 467         hw->bus.device = pci_get_slot(dev);
 468         hw->bus.func = pci_get_function(dev);
 469
 470         pf->vc_debug_lvl = 1;
 471
 472         /* Do PCI setup - map BAR0, etc */
 473         if (ixl_allocate_pci_resources(pf)) {
 474                 device_printf(dev, "Allocation of PCI resources failed\n");
 475                 error = ENXIO;
 476                 goto err_out;
 477         }
 478
 479         /* Establish a clean starting point */
 480         i40e_clear_hw(hw);
 481         error = i40e_pf_reset(hw);
 482         if (error) {
 483                 device_printf(dev, "PF reset failure %d\n", error);
 484                 error = EIO;
 485                 goto err_out;
 486         }
 487
 488         /* Set admin queue parameters */
 489         hw->aq.num_arq_entries = IXL_AQ_LEN;
 490         hw->aq.num_asq_entries = IXL_AQ_LEN;
 491         hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
 492         hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
 493
 494         /* Initialize mac filter list for VSI */
 495         SLIST_INIT(&vsi->ftl);
 496
 497         /* Initialize the shared code */
 498         error = i40e_init_shared_code(hw);
 499         if (error) {
 500                 device_printf(dev, "Unable to initialize shared code, error %d\n",
 501                     error);
 502                 error = EIO;
 503                 goto err_out;
 504         }
 505
 506         /* Set up the admin queue */
 507         error = i40e_init_adminq(hw);
 508         if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
 509                 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
 510                     error);
 511                 error = EIO;
 512                 goto err_out;
 513         }
 514         ixl_print_nvm_version(pf);
 515
 516         if (error == I40E_ERR_FIRMWARE_API_VERSION) {
 517                 device_printf(dev, "The driver for the device stopped "
 518                     "because the NVM image is newer than expected.\n"
 519                     "You must install the most recent version of "
 520                     "the network driver.\n");
 521                 error = EIO;
 522                 goto err_out;
 523         }
 524
 525         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
 526             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
 527                 device_printf(dev, "The driver for the device detected "
 528                     "a newer version of the NVM image than expected.\n"
 529                     "Please install the most recent version of the network driver.\n");
 530         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
 531             hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
 532                 device_printf(dev, "The driver for the device detected "
 533                     "an older version of the NVM image than expected.\n"
 534                     "Please update the NVM image.\n");
 535
 536         /* Clear PXE mode */
 537         i40e_clear_pxe_mode(hw);
 538
 539         /* Get capabilities from the device */
 540         error = ixl_get_hw_capabilities(pf);
 541         if (error) {
 542                 device_printf(dev, "HW capabilities failure!\n");
 543                 goto err_get_cap;
 544         }
 545
 546         /* Set up host memory cache */
 547         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 548             hw->func_caps.num_rx_qp, 0, 0);
 549         if (error) {
 550                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
 551                 goto err_get_cap;
 552         }
 553
 554         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 555         if (error) {
 556                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
 557                 goto err_mac_hmc;
 558         }
 559
 560         /* Disable LLDP from the firmware for certain NVM versions */
 561         if (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
 562             (pf->hw.aq.fw_maj_ver < 4))
 563                 i40e_aq_stop_lldp(hw, TRUE, NULL);
 564
 565         i40e_get_mac_addr(hw, hw->mac.addr);
 566         error = i40e_validate_mac_addr(hw->mac.addr);
 567         if (error) {
 568                 device_printf(dev, "validate_mac_addr failed: %d\n", error);
 569                 goto err_mac_hmc;
 570         }
 571         bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
 572         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 573
 574         /* Set up VSI and queues */
 575         if (ixl_setup_stations(pf) != 0) {
 576                 device_printf(dev, "setup stations failed!\n");
 577                 error = ENOMEM;
 578                 goto err_mac_hmc;
 579         }
 580
 581         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 582             (hw->aq.fw_maj_ver < 4)) {
 583                 i40e_msec_delay(75);
 584                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 585                 if (error) {
 586                         device_printf(dev, "link restart failed, aq_err=%d\n",
 587                             pf->hw.aq.asq_last_status);
 588                         goto err_late;
 589                 }
 590         }
 591
 592         /* Determine link state */
 593         hw->phy.get_link_info = TRUE;
 594         i40e_get_link_status(hw, &pf->link_up);
 595
 596         /* Setup OS network interface / ifnet */
 597         if (ixl_setup_interface(dev, vsi) != 0) {
 598                 device_printf(dev, "interface setup failed!\n");
 599                 error = EIO;
 600                 goto err_late;
 601         }
 602
 603         error = ixl_switch_config(pf);
 604         if (error) {
 605                 device_printf(dev, "Initial ixl_switch_config() failed: %d\n",
 606                      error);
 607                 goto err_late;
 608         }
 609
 610         /* Limit PHY interrupts to link, autoneg, and modules failure */
 611         error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
 612             NULL);
 613         if (error) {
 614                 device_printf(dev, "i40e_aq_set_phy_mask() failed: err %d,"
 615                     " aq_err %d\n", error, hw->aq.asq_last_status);
 616                 goto err_late;
 617         }
 618
 619         /* Get the bus configuration and set the shared code's config */
 620         bus = ixl_get_bus_info(hw, dev);
 621         i40e_set_pci_config_data(hw, bus);
 622
 623         /*
 624          * In MSI-X mode, initialize the Admin Queue interrupt,
 625          * so userland tools can communicate with the adapter regardless of
 626          * the ifnet interface's status.
 627          */
 628         if (pf->msix > 1) {
 629                 error = ixl_setup_adminq_msix(pf);
 630                 if (error) {
 631                         device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
 632                             error);
 633                         goto err_late;
 634                 }
 635                 error = ixl_setup_adminq_tq(pf);
 636                 if (error) {
 637                         device_printf(dev, "ixl_setup_adminq_tq error: %d\n",
 638                             error);
 639                         goto err_late;
 640                 }
 641                 ixl_configure_intr0_msix(pf);
 642                 ixl_enable_adminq(hw);
 643         }
 644
 645         /* Initialize statistics & add sysctls */
 646         ixl_add_device_sysctls(pf);
 647
 648         ixl_pf_reset_stats(pf);
 649         ixl_update_stats_counters(pf);
 650         ixl_add_hw_stats(pf);
 651
 652         /* Register for VLAN events */
 653         vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 654             ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 655         vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 656             ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 657
 658 #ifdef PCI_IOV
 659         /* SR-IOV is only supported when MSI-X is in use. */
 660         if (pf->msix > 1) {
 661                 pf_schema = pci_iov_schema_alloc_node();
 662                 vf_schema = pci_iov_schema_alloc_node();
 663                 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
 664                 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
 665                     IOV_SCHEMA_HASDEFAULT, TRUE);
 666                 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
 667                     IOV_SCHEMA_HASDEFAULT, FALSE);
 668                 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
 669                     IOV_SCHEMA_HASDEFAULT, FALSE);
 670
 671                 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
 672                 if (iov_error != 0) {
 673                         device_printf(dev,
 674                             "Failed to initialize SR-IOV (error=%d)\n",
 675                             iov_error);
 676                 } else
 677                         device_printf(dev, "SR-IOV ready\n");
 678         }
 679 #endif
 680
 681 #ifdef DEV_NETMAP
 682         ixl_netmap_attach(vsi);
 683 #endif /* DEV_NETMAP */
 684         INIT_DEBUGOUT("ixl_attach: end");
 685         return (0);
 686
 687 err_late:
 688         if (vsi->ifp != NULL)
 689                 if_free(vsi->ifp);
 690 err_mac_hmc:
 691         i40e_shutdown_lan_hmc(hw);
 692 err_get_cap:
 693         i40e_shutdown_adminq(hw);
 694 err_out:
 695         ixl_free_pci_resources(pf);
 696         ixl_free_vsi(vsi);
 697         IXL_PF_LOCK_DESTROY(pf);
 698         return (error);
 699 }
 700
 701 /*********************************************************************
 702  *  Device removal routine
 703  *
 704  *  The detach entry point is called when the driver is being removed.
 705  *  This routine stops the adapter and deallocates all the resources
 706  *  that were allocated for driver operation.
 707  *
 708  *  return 0 on success, positive on failure
 709  *********************************************************************/
 710
 711 static int
 712 ixl_detach(device_t dev)
 713 {
 714         struct ixl_pf           *pf = device_get_softc(dev);
 715         struct i40e_hw          *hw = &pf->hw;
 716         struct ixl_vsi          *vsi = &pf->vsi;
 717         enum i40e_status_code   status;
 718 #ifdef PCI_IOV
 719         int                     error;
 720 #endif
 721
 722         INIT_DEBUGOUT("ixl_detach: begin");
 723
 724         /* Make sure VLANS are not using driver */
 725         if (vsi->ifp->if_vlantrunk != NULL) {
 726                 device_printf(dev, "Vlan in use, detach first\n");
 727                 return (EBUSY);
 728         }
 729
 730 #ifdef PCI_IOV
 731         error = pci_iov_detach(dev);
 732         if (error != 0) {
 733                 device_printf(dev, "SR-IOV in use; detach first.\n");
 734                 return (error);
 735         }
 736 #endif
 737
 738         ether_ifdetach(vsi->ifp);
 739         if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)
 740                 ixl_stop(pf);
 741
 742         ixl_free_queue_tqs(vsi);
 743
 744         /* Shutdown LAN HMC */
 745         status = i40e_shutdown_lan_hmc(hw);
 746         if (status)
 747                 device_printf(dev,
 748                     "Shutdown LAN HMC failed with code %d\n", status);
 749
 750         /* Shutdown admin queue */
 751         ixl_disable_adminq(hw);
 752         ixl_free_adminq_tq(pf);
 753         ixl_teardown_adminq_msix(pf);
 754         status = i40e_shutdown_adminq(hw);
 755         if (status)
 756                 device_printf(dev,
 757                     "Shutdown Admin queue failed with code %d\n", status);
 758
 759         /* Unregister VLAN events */
 760         if (vsi->vlan_attach != NULL)
 761                 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
 762         if (vsi->vlan_detach != NULL)
 763                 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
 764
 765         callout_drain(&pf->timer);
 766 #ifdef DEV_NETMAP
 767         netmap_detach(vsi->ifp);
 768 #endif /* DEV_NETMAP */
 769         ixl_free_pci_resources(pf);
 770         bus_generic_detach(dev);
 771         if_free(vsi->ifp);
 772         ixl_free_vsi(vsi);
 773         IXL_PF_LOCK_DESTROY(pf);
 774         return (0);
 775 }
 776
 777 /*********************************************************************
 778  *
 779  *  Shutdown entry point
 780  *
 781  **********************************************************************/
 782
 783 static int
 784 ixl_shutdown(device_t dev)
 785 {
 786         struct ixl_pf *pf = device_get_softc(dev);
 787         ixl_stop(pf);
 788         return (0);
 789 }
 790
 791
 792 /*********************************************************************
 793  *
 794  *  Get the hardware capabilities
 795  *
 796  **********************************************************************/
 797
 798 static int
 799 ixl_get_hw_capabilities(struct ixl_pf *pf)
 800 {
 801         struct i40e_aqc_list_capabilities_element_resp *buf;
 802         struct i40e_hw  *hw = &pf->hw;
 803         device_t        dev = pf->dev;
 804         int             error, len;
 805         u16             needed;
 806         bool            again = TRUE;
 807
 808         len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
 809 retry:
 810         if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
 811             malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 812                 device_printf(dev, "Unable to allocate cap memory\n");
 813                 return (ENOMEM);
 814         }
 815
 816         /* This populates the hw struct */
 817         error = i40e_aq_discover_capabilities(hw, buf, len,
 818             &needed, i40e_aqc_opc_list_func_capabilities, NULL);
 819         free(buf, M_DEVBUF);
 820         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
 821             (again == TRUE)) {
 822                 /* retry once with a larger buffer */
 823                 again = FALSE;
 824                 len = needed;
 825                 goto retry;
 826         } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
 827                 device_printf(dev, "capability discovery failed: %d\n",
 828                     pf->hw.aq.asq_last_status);
 829                 return (ENODEV);
 830         }
 831
 832         /* Capture this PF's starting queue pair */
 833         pf->qbase = hw->func_caps.base_queue;
 834
 835 #ifdef IXL_DEBUG
 836         device_printf(dev, "pf_id=%d, num_vfs=%d, msix_pf=%d, "
 837             "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
 838             hw->pf_id, hw->func_caps.num_vfs,
 839             hw->func_caps.num_msix_vectors,
 840             hw->func_caps.num_msix_vectors_vf,
 841             hw->func_caps.fd_filters_guaranteed,
 842             hw->func_caps.fd_filters_best_effort,
 843             hw->func_caps.num_tx_qp,
 844             hw->func_caps.num_rx_qp,
 845             hw->func_caps.base_queue);
 846 #endif
 847         return (error);
 848 }
 849
 850 static void
 851 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
 852 {
 853         device_t        dev = vsi->dev;
 854
 855         /* Enable/disable TXCSUM/TSO4 */
 856         if (!(ifp->if_capenable & IFCAP_TXCSUM)
 857             && !(ifp->if_capenable & IFCAP_TSO4)) {
 858                 if (mask & IFCAP_TXCSUM) {
 859                         ifp->if_capenable |= IFCAP_TXCSUM;
 860                         /* enable TXCSUM, restore TSO if previously enabled */
 861                         if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
 862                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 863                                 ifp->if_capenable |= IFCAP_TSO4;
 864                         }
 865                 }
 866                 else if (mask & IFCAP_TSO4) {
 867                         ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
 868                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 869                         device_printf(dev,
 870                             "TSO4 requires txcsum, enabling both...\n");
 871                 }
 872         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 873             && !(ifp->if_capenable & IFCAP_TSO4)) {
 874                 if (mask & IFCAP_TXCSUM)
 875                         ifp->if_capenable &= ~IFCAP_TXCSUM;
 876                 else if (mask & IFCAP_TSO4)
 877                         ifp->if_capenable |= IFCAP_TSO4;
 878         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 879             && (ifp->if_capenable & IFCAP_TSO4)) {
 880                 if (mask & IFCAP_TXCSUM) {
 881                         vsi->flags |= IXL_FLAGS_KEEP_TSO4;
 882                         ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
 883                         device_printf(dev,
 884                             "TSO4 requires txcsum, disabling both...\n");
 885                 } else if (mask & IFCAP_TSO4)
 886                         ifp->if_capenable &= ~IFCAP_TSO4;
 887         }
 888
 889         /* Enable/disable TXCSUM_IPV6/TSO6 */
 890         if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 891             && !(ifp->if_capenable & IFCAP_TSO6)) {
 892                 if (mask & IFCAP_TXCSUM_IPV6) {
 893                         ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
 894                         if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
 895                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 896                                 ifp->if_capenable |= IFCAP_TSO6;
 897                         }
 898                 } else if (mask & IFCAP_TSO6) {
 899                         ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 900                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 901                         device_printf(dev,
 902                             "TSO6 requires txcsum6, enabling both...\n");
 903                 }
 904         } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 905             && !(ifp->if_capenable & IFCAP_TSO6)) {
 906                 if (mask & IFCAP_TXCSUM_IPV6)
 907                         ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
 908                 else if (mask & IFCAP_TSO6)
 909                         ifp->if_capenable |= IFCAP_TSO6;
 910         } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 911             && (ifp->if_capenable & IFCAP_TSO6)) {
 912                 if (mask & IFCAP_TXCSUM_IPV6) {
 913                         vsi->flags |= IXL_FLAGS_KEEP_TSO6;
 914                         ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 915                         device_printf(dev,
 916                             "TSO6 requires txcsum6, disabling both...\n");
 917                 } else if (mask & IFCAP_TSO6)
 918                         ifp->if_capenable &= ~IFCAP_TSO6;
 919         }
 920 }
 921
 922 /*********************************************************************
 923  *  Ioctl entry point
 924  *
 925  *  ixl_ioctl is called when the user wants to configure the
 926  *  interface.
 927  *
 928  *  return 0 on success, positive on failure
 929  **********************************************************************/
 930
 931 static int
 932 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 933 {
 934         struct ixl_vsi  *vsi = ifp->if_softc;
 935         struct ixl_pf   *pf = vsi->back;
 936         struct ifreq    *ifr = (struct ifreq *)data;
 937         struct ifdrv    *ifd = (struct ifdrv *)data;
 938 #if defined(INET) || defined(INET6)
 939         struct ifaddr *ifa = (struct ifaddr *)data;
 940         bool            avoid_reset = FALSE;
 941 #endif
 942         int             error = 0;
 943
 944         switch (command) {
 945
 946         case SIOCSIFADDR:
 947 #ifdef INET
 948                 if (ifa->ifa_addr->sa_family == AF_INET)
 949                         avoid_reset = TRUE;
 950 #endif
 951 #ifdef INET6
 952                 if (ifa->ifa_addr->sa_family == AF_INET6)
 953                         avoid_reset = TRUE;
 954 #endif
 955 #if defined(INET) || defined(INET6)
 956                 /*
 957                 ** Calling init results in link renegotiation,
 958                 ** so we avoid doing it when possible.
 959                 */
 960                 if (avoid_reset) {
 961                         ifp->if_flags |= IFF_UP;
 962                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 963                                 ixl_init(pf);
 964 #ifdef INET
 965                         if (!(ifp->if_flags & IFF_NOARP))
 966                                 arp_ifinit(ifp, ifa);
 967 #endif
 968                 } else
 969                         error = ether_ioctl(ifp, command, data);
 970                 break;
 971 #endif
 972         case SIOCSIFMTU:
 973                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 974                 if (ifr->ifr_mtu > IXL_MAX_FRAME -
 975                    ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
 976                         error = EINVAL;
 977                 } else {
 978                         IXL_PF_LOCK(pf);
 979                         ifp->if_mtu = ifr->ifr_mtu;
 980                         vsi->max_frame_size =
 981                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
 982                             + ETHER_VLAN_ENCAP_LEN;
 983                         ixl_init_locked(pf);
 984                         IXL_PF_UNLOCK(pf);
 985                 }
 986                 break;
 987         case SIOCSIFFLAGS:
 988                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 989                 IXL_PF_LOCK(pf);
 990                 if (ifp->if_flags & IFF_UP) {
 991                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 992                                 if ((ifp->if_flags ^ pf->if_flags) &
 993                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 994                                         ixl_set_promisc(vsi);
 995                                 }
 996                         } else {
 997                                 IXL_PF_UNLOCK(pf);
 998                                 ixl_init(pf);
 999                                 IXL_PF_LOCK(pf);
1000                         }
1001                 } else {
1002                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1003                                 IXL_PF_UNLOCK(pf);
1004                                 ixl_stop(pf);
1005                                 IXL_PF_LOCK(pf);
1006                         }
1007                 }
1008                 pf->if_flags = ifp->if_flags;
1009                 IXL_PF_UNLOCK(pf);
1010                 break;
1011         case SIOCSDRVSPEC:
1012         case SIOCGDRVSPEC:
1013                 IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific "
1014                     "Info)\n");
1015
1016                 /* NVM update command */
1017                 if (ifd->ifd_cmd == I40E_NVM_ACCESS)
1018                         error = ixl_handle_nvmupd_cmd(pf, ifd);
1019                 else
1020                         error = EINVAL;
1021                 break;
1022         case SIOCADDMULTI:
1023                 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1024                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1025                         IXL_PF_LOCK(pf);
1026                         ixl_disable_intr(vsi);
1027                         ixl_add_multi(vsi);
1028                         ixl_enable_intr(vsi);
1029                         IXL_PF_UNLOCK(pf);
1030                 }
1031                 break;
1032         case SIOCDELMULTI:
1033                 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1034                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1035                         IXL_PF_LOCK(pf);
1036                         ixl_disable_intr(vsi);
1037                         ixl_del_multi(vsi);
1038                         ixl_enable_intr(vsi);
1039                         IXL_PF_UNLOCK(pf);
1040                 }
1041                 break;
1042         case SIOCSIFMEDIA:
1043         case SIOCGIFMEDIA:
1044 #ifdef IFM_ETH_XTYPE
1045         case SIOCGIFXMEDIA:
1046 #endif
1047                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1048                 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1049                 break;
1050         case SIOCSIFCAP:
1051         {
1052                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1053                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1054
1055                 ixl_cap_txcsum_tso(vsi, ifp, mask);
1056
1057                 if (mask & IFCAP_RXCSUM)
1058                         ifp->if_capenable ^= IFCAP_RXCSUM;
1059                 if (mask & IFCAP_RXCSUM_IPV6)
1060                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1061                 if (mask & IFCAP_LRO)
1062                         ifp->if_capenable ^= IFCAP_LRO;
1063                 if (mask & IFCAP_VLAN_HWTAGGING)
1064                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1065                 if (mask & IFCAP_VLAN_HWFILTER)
1066                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1067                 if (mask & IFCAP_VLAN_HWTSO)
1068                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1069                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1070                         IXL_PF_LOCK(pf);
1071                         ixl_init_locked(pf);
1072                         IXL_PF_UNLOCK(pf);
1073                 }
1074                 VLAN_CAPABILITIES(ifp);
1075
1076                 break;
1077         }
1078
1079         default:
1080                 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1081                 error = ether_ioctl(ifp, command, data);
1082                 break;
1083         }
1084
1085         return (error);
1086 }
1087
1088
1089 /*********************************************************************
1090  *  Init entry point
1091  *
1092  *  This routine is used in two ways. It is used by the stack as
1093  *  init entry point in network interface structure. It is also used
1094  *  by the driver as a hw/sw initialization routine to get to a
1095  *  consistent state.
1096  *
1097  *  return 0 on success, positive on failure
1098  **********************************************************************/
1099
1100 static void
1101 ixl_init_locked(struct ixl_pf *pf)
1102 {
1103         struct i40e_hw  *hw = &pf->hw;
1104         struct ixl_vsi  *vsi = &pf->vsi;
1105         struct ifnet    *ifp = vsi->ifp;
1106         device_t        dev = pf->dev;
1107         struct i40e_filter_control_settings     filter;
1108         u8              tmpaddr[ETHER_ADDR_LEN];
1109         int             ret;
1110
1111         mtx_assert(&pf->pf_mtx, MA_OWNED);
1112         INIT_DEBUGOUT("ixl_init_locked: begin");
1113
1114         ixl_stop_locked(pf);
1115
1116         /* Get the latest mac address... User might use a LAA */
1117         bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1118               I40E_ETH_LENGTH_OF_ADDRESS);
1119         if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1120             (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
1121                 ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1122                 bcopy(tmpaddr, hw->mac.addr,
1123                     I40E_ETH_LENGTH_OF_ADDRESS);
1124                 ret = i40e_aq_mac_address_write(hw,
1125                     I40E_AQC_WRITE_TYPE_LAA_ONLY,
1126                     hw->mac.addr, NULL);
1127                 if (ret) {
1128                         device_printf(dev, "LLA address"
1129                          "change failed!!\n");
1130                         return;
1131                 }
1132         }
1133
1134         ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1135
1136         /* Set the various hardware offload abilities */
1137         ifp->if_hwassist = 0;
1138         if (ifp->if_capenable & IFCAP_TSO)
1139                 ifp->if_hwassist |= CSUM_TSO;
1140         if (ifp->if_capenable & IFCAP_TXCSUM)
1141                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1142         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1143                 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1144
1145         /* Set up the device filtering */
1146         bzero(&filter, sizeof(filter));
1147         filter.enable_ethtype = TRUE;
1148         filter.enable_macvlan = TRUE;
1149 #ifdef IXL_FDIR
1150         filter.enable_fdir = TRUE;
1151 #endif
1152         filter.hash_lut_size = I40E_HASH_LUT_SIZE_512;
1153         if (i40e_set_filter_control(hw, &filter))
1154                 device_printf(dev, "i40e_set_filter_control() failed\n");
1155
1156         /* Set up RSS */
1157         ixl_config_rss(vsi);
1158
1159         /* Prepare the VSI: rings, hmc contexts, etc... */
1160         if (ixl_initialize_vsi(vsi)) {
1161                 device_printf(dev, "initialize vsi failed!!\n");
1162                 return;
1163         }
1164
1165         /* Add protocol filters to list */
1166         ixl_init_filters(vsi);
1167
1168         /* Setup vlan's if needed */
1169         ixl_setup_vlan_filters(vsi);
1170
1171         /* Set up MSI/X routing and the ITR settings */
1172         if (ixl_enable_msix) {
1173                 ixl_configure_queue_intr_msix(pf);
1174                 ixl_configure_itr(pf);
1175         } else
1176                 ixl_configure_legacy(pf);
1177
1178         ixl_enable_rings(vsi);
1179
1180         i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1181
1182         ixl_reconfigure_filters(vsi);
1183
1184         /* And now turn on interrupts */
1185         ixl_enable_intr(vsi);
1186
1187         /* Get link info */
1188         hw->phy.get_link_info = TRUE;
1189         i40e_get_link_status(hw, &pf->link_up);
1190         ixl_update_link_status(pf);
1191
1192         /* Set initial advertised speed sysctl value */
1193         ixl_get_initial_advertised_speeds(pf);
1194
1195         /* Start the local timer */
1196         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1197
1198         /* Now inform the stack we're ready */
1199         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1200
1201         return;
1202 }
1203
1204 /* For the set_advertise sysctl */
1205 static void
1206 ixl_get_initial_advertised_speeds(struct ixl_pf *pf)
1207 {
1208         struct i40e_hw *hw = &pf->hw;
1209         device_t dev = pf->dev;
1210         enum i40e_status_code status;
1211         struct i40e_aq_get_phy_abilities_resp abilities;
1212
1213         /* Set initial sysctl values */
1214         status = i40e_aq_get_phy_capabilities(hw, FALSE, false, &abilities,
1215                                               NULL);
1216         if (status) {
1217                 /* Non-fatal error */
1218                 device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n",
1219                      __func__, status);
1220                 return;
1221         }
1222
1223         if (abilities.link_speed & I40E_LINK_SPEED_40GB)
1224                 pf->advertised_speed |= 0x10;
1225         if (abilities.link_speed & I40E_LINK_SPEED_20GB)
1226                 pf->advertised_speed |= 0x8;
1227         if (abilities.link_speed & I40E_LINK_SPEED_10GB)
1228                 pf->advertised_speed |= 0x4;
1229         if (abilities.link_speed & I40E_LINK_SPEED_1GB)
1230                 pf->advertised_speed |= 0x2;
1231         if (abilities.link_speed & I40E_LINK_SPEED_100MB)
1232                 pf->advertised_speed |= 0x1;
1233 }
1234
1235 static int
1236 ixl_teardown_hw_structs(struct ixl_pf *pf)
1237 {
1238         enum i40e_status_code status = 0;
1239         struct i40e_hw *hw = &pf->hw;
1240         device_t dev = pf->dev;
1241
1242         /* Shutdown LAN HMC */
1243         if (hw->hmc.hmc_obj) {
1244                 status = i40e_shutdown_lan_hmc(hw);
1245                 if (status) {
1246                         device_printf(dev,
1247                             "init: LAN HMC shutdown failure; status %d\n", status);
1248                         goto err_out;
1249                 }
1250         }
1251
1252         // XXX: This gets called when we know the adminq is inactive;
1253         // so we already know it's setup when we get here.
1254
1255         /* Shutdown admin queue */
1256         status = i40e_shutdown_adminq(hw);
1257         if (status)
1258                 device_printf(dev,
1259                     "init: Admin Queue shutdown failure; status %d\n", status);
1260
1261 err_out:
1262         return (status);
1263 }
1264
1265 static int
1266 ixl_reset(struct ixl_pf *pf)
1267 {
1268         struct i40e_hw *hw = &pf->hw;
1269         device_t dev = pf->dev;
1270         u8 set_fc_err_mask;
1271         int error = 0;
1272
1273         // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary
1274         i40e_clear_hw(hw);
1275         error = i40e_pf_reset(hw);
1276         if (error) {
1277                 device_printf(dev, "init: PF reset failure");
1278                 error = EIO;
1279                 goto err_out;
1280         }
1281
1282         error = i40e_init_adminq(hw);
1283         if (error) {
1284                 device_printf(dev, "init: Admin queue init failure;"
1285                     " status code %d", error);
1286                 error = EIO;
1287                 goto err_out;
1288         }
1289
1290         i40e_clear_pxe_mode(hw);
1291
1292         error = ixl_get_hw_capabilities(pf);
1293         if (error) {
1294                 device_printf(dev, "init: Error retrieving HW capabilities;"
1295                     " status code %d\n", error);
1296                 goto err_out;
1297         }
1298
1299         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1300             hw->func_caps.num_rx_qp, 0, 0);
1301         if (error) {
1302                 device_printf(dev, "init: LAN HMC init failed; status code %d\n",
1303                     error);
1304                 error = EIO;
1305                 goto err_out;
1306         }
1307
1308         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1309         if (error) {
1310                 device_printf(dev, "init: LAN HMC config failed; status code %d\n",
1311                     error);
1312                 error = EIO;
1313                 goto err_out;
1314         }
1315
1316         // XXX: possible fix for panic, but our failure recovery is still broken
1317         error = ixl_switch_config(pf);
1318         if (error) {
1319                 device_printf(dev, "init: ixl_switch_config() failed: %d\n",
1320                      error);
1321                 goto err_out;
1322         }
1323
1324         error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
1325             NULL);
1326         if (error) {
1327                 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
1328                     " aq_err %d\n", error, hw->aq.asq_last_status);
1329                 error = EIO;
1330                 goto err_out;
1331         }
1332
1333         error = i40e_set_fc(hw, &set_fc_err_mask, true);
1334         if (error) {
1335                 device_printf(dev, "init: setting link flow control failed; retcode %d,"
1336                     " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
1337                 goto err_out;
1338         }
1339
1340         // XXX: (Rebuild VSIs?)
1341
1342         /* Firmware delay workaround */
1343         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
1344             (hw->aq.fw_maj_ver < 4)) {
1345                 i40e_msec_delay(75);
1346                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
1347                 if (error) {
1348                         device_printf(dev, "init: link restart failed, aq_err %d\n",
1349                             hw->aq.asq_last_status);
1350                         goto err_out;
1351                 }
1352         }
1353
1354
1355 err_out:
1356         return (error);
1357 }
1358
1359 static void
1360 ixl_init(void *arg)
1361 {
1362         struct ixl_pf *pf = arg;
1363         struct ixl_vsi *vsi = &pf->vsi;
1364         device_t dev = pf->dev;
1365         int error = 0;
1366
1367         /*
1368          * If the aq is dead here, it probably means something outside of the driver
1369          * did something to the adapter, like a PF reset.
1370          * So rebuild the driver's state here if that occurs.
1371          */
1372         if (!i40e_check_asq_alive(&pf->hw)) {
1373                 device_printf(dev, "Admin Queue is down; resetting...\n");
1374                 IXL_PF_LOCK(pf);
1375                 ixl_teardown_hw_structs(pf);
1376                 ixl_reset(pf);
1377                 IXL_PF_UNLOCK(pf);
1378         }
1379
1380         /*
1381          * Set up LAN queue interrupts here.
1382          * Kernel interrupt setup functions cannot be called while holding a lock,
1383          * so this is done outside of init_locked().
1384          */
1385         if (pf->msix > 1) {
1386                 /* Teardown existing interrupts, if they exist */
1387                 ixl_teardown_queue_msix(vsi);
1388                 ixl_free_queue_tqs(vsi);
1389                 /* Then set them up again */
1390                 error = ixl_setup_queue_msix(vsi);
1391                 if (error)
1392                         device_printf(dev, "ixl_setup_queue_msix() error: %d\n",
1393                             error);
1394                 error = ixl_setup_queue_tqs(vsi);
1395                 if (error)
1396                         device_printf(dev, "ixl_setup_queue_tqs() error: %d\n",
1397                             error);
1398         } else
1399                 // possibly broken
1400                 error = ixl_assign_vsi_legacy(pf);
1401         if (error) {
1402                 device_printf(pf->dev, "assign_vsi_msix/legacy error: %d\n", error);
1403                 return;
1404         }
1405
1406         IXL_PF_LOCK(pf);
1407         ixl_init_locked(pf);
1408         IXL_PF_UNLOCK(pf);
1409 }
1410
1411 /*
1412 ** MSIX Interrupt Handlers and Tasklets
1413 */
1414 static void
1415 ixl_handle_que(void *context, int pending)
1416 {
1417         struct ixl_queue *que = context;
1418         struct ixl_vsi *vsi = que->vsi;
1419         struct i40e_hw  *hw = vsi->hw;
1420         struct tx_ring  *txr = &que->txr;
1421         struct ifnet    *ifp = vsi->ifp;
1422         bool            more;
1423
1424         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1425                 more = ixl_rxeof(que, IXL_RX_LIMIT);
1426                 IXL_TX_LOCK(txr);
1427                 ixl_txeof(que);
1428                 if (!drbr_empty(ifp, txr->br))
1429                         ixl_mq_start_locked(ifp, txr);
1430                 IXL_TX_UNLOCK(txr);
1431                 if (more) {
1432                         taskqueue_enqueue(que->tq, &que->task);
1433                         return;
1434                 }
1435         }
1436
1437         /* Reenable this interrupt - hmmm */
1438         ixl_enable_queue(hw, que->me);
1439         return;
1440 }
1441
1442
1443 /*********************************************************************
1444  *
1445  *  Legacy Interrupt Service routine
1446  *
1447  **********************************************************************/
1448 void
1449 ixl_intr(void *arg)
1450 {
1451         struct ixl_pf           *pf = arg;
1452         struct i40e_hw          *hw =  &pf->hw;
1453         struct ixl_vsi          *vsi = &pf->vsi;
1454         struct ixl_queue        *que = vsi->queues;
1455         struct ifnet            *ifp = vsi->ifp;
1456         struct tx_ring          *txr = &que->txr;
1457         u32                     reg, icr0, mask;
1458         bool                    more_tx, more_rx;
1459
1460         ++que->irqs;
1461
1462         /* Protect against spurious interrupts */
1463         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1464                 return;
1465
1466         icr0 = rd32(hw, I40E_PFINT_ICR0);
1467
1468         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1469         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1470         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1471
1472         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1473
1474 #ifdef PCI_IOV
1475         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1476                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1477 #endif
1478
1479         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1480                 taskqueue_enqueue(pf->tq, &pf->adminq);
1481                 return;
1482         }
1483
1484         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1485
1486         IXL_TX_LOCK(txr);
1487         more_tx = ixl_txeof(que);
1488         if (!drbr_empty(vsi->ifp, txr->br))
1489                 more_tx = 1;
1490         IXL_TX_UNLOCK(txr);
1491
1492         /* re-enable other interrupt causes */
1493         wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1494
1495         /* And now the queues */
1496         reg = rd32(hw, I40E_QINT_RQCTL(0));
1497         reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1498         wr32(hw, I40E_QINT_RQCTL(0), reg);
1499
1500         reg = rd32(hw, I40E_QINT_TQCTL(0));
1501         reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1502         reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1503         wr32(hw, I40E_QINT_TQCTL(0), reg);
1504
1505         ixl_enable_legacy(hw);
1506
1507         return;
1508 }
1509
1510
1511 /*********************************************************************
1512  *
1513  *  MSIX VSI Interrupt Service routine
1514  *
1515  **********************************************************************/
1516 void
1517 ixl_msix_que(void *arg)
1518 {
1519         struct ixl_queue        *que = arg;
1520         struct ixl_vsi  *vsi = que->vsi;
1521         struct i40e_hw  *hw = vsi->hw;
1522         struct tx_ring  *txr = &que->txr;
1523         bool            more_tx, more_rx;
1524
1525         /* Protect against spurious interrupts */
1526         if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1527                 return;
1528
1529         ++que->irqs;
1530
1531         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1532
1533         IXL_TX_LOCK(txr);
1534         more_tx = ixl_txeof(que);
1535         /*
1536         ** Make certain that if the stack
1537         ** has anything queued the task gets
1538         ** scheduled to handle it.
1539         */
1540         if (!drbr_empty(vsi->ifp, txr->br))
1541                 more_tx = 1;
1542         IXL_TX_UNLOCK(txr);
1543
1544         ixl_set_queue_rx_itr(que);
1545         ixl_set_queue_tx_itr(que);
1546
1547         if (more_tx || more_rx)
1548                 taskqueue_enqueue(que->tq, &que->task);
1549         else
1550                 ixl_enable_queue(hw, que->me);
1551
1552         return;
1553 }
1554
1555
1556 /*********************************************************************
1557  *
1558  *  MSIX Admin Queue Interrupt Service routine
1559  *
1560  **********************************************************************/
1561 static void
1562 ixl_msix_adminq(void *arg)
1563 {
1564         struct ixl_pf   *pf = arg;
1565         struct i40e_hw  *hw = &pf->hw;
1566         u32             reg, mask, rstat_reg;
1567         bool            do_task = FALSE;
1568
1569         ++pf->admin_irq;
1570
1571         reg = rd32(hw, I40E_PFINT_ICR0);
1572         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1573
1574         /* Check on the cause */
1575         if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) {
1576                 mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK;
1577                 do_task = TRUE;
1578         }
1579
1580         if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1581                 ixl_handle_mdd_event(pf);
1582                 mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK;
1583         }
1584
1585         if (reg & I40E_PFINT_ICR0_GRST_MASK) {
1586                 device_printf(pf->dev, "Reset Requested!\n");
1587                 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT);
1588                 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
1589                     >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
1590                 device_printf(pf->dev, "Reset type: ");
1591                 switch (rstat_reg) {
1592                 /* These others might be handled similarly to an EMPR reset */
1593                 case I40E_RESET_CORER:
1594                         printf("CORER\n");
1595                         break;
1596                 case I40E_RESET_GLOBR:
1597                         printf("GLOBR\n");
1598                         break;
1599                 case I40E_RESET_EMPR:
1600                         printf("EMPR\n");
1601                         atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
1602                         break;
1603                 default:
1604                         printf("?\n");
1605                         break;
1606                 }
1607                 // overload admin queue task to check reset progress?
1608                 do_task = TRUE;
1609         }
1610
1611         if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) {
1612                 device_printf(pf->dev, "ECC Error detected!\n");
1613         }
1614
1615         if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) {
1616                 device_printf(pf->dev, "HMC Error detected!\n");
1617         }
1618
1619         if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) {
1620                 device_printf(pf->dev, "PCI Exception detected!\n");
1621         }
1622
1623 #ifdef PCI_IOV
1624         if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1625                 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1626                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1627         }
1628 #endif
1629
1630         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1631         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1632         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1633
1634         if (do_task)
1635                 taskqueue_enqueue(pf->tq, &pf->adminq);
1636 }
1637
1638 /*********************************************************************
1639  *
1640  *  Media Ioctl callback
1641  *
1642  *  This routine is called whenever the user queries the status of
1643  *  the interface using ifconfig.
1644  *
1645  **********************************************************************/
1646 static void
1647 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1648 {
1649         struct ixl_vsi  *vsi = ifp->if_softc;
1650         struct ixl_pf   *pf = vsi->back;
1651         struct i40e_hw  *hw = &pf->hw;
1652
1653         INIT_DEBUGOUT("ixl_media_status: begin");
1654         IXL_PF_LOCK(pf);
1655
1656         hw->phy.get_link_info = TRUE;
1657         i40e_get_link_status(hw, &pf->link_up);
1658         ixl_update_link_status(pf);
1659
1660         ifmr->ifm_status = IFM_AVALID;
1661         ifmr->ifm_active = IFM_ETHER;
1662
1663         if (!pf->link_up) {
1664                 IXL_PF_UNLOCK(pf);
1665                 return;
1666         }
1667
1668         ifmr->ifm_status |= IFM_ACTIVE;
1669
1670         /* Hardware always does full-duplex */
1671         ifmr->ifm_active |= IFM_FDX;
1672
1673         switch (hw->phy.link_info.phy_type) {
1674                 /* 100 M */
1675                 case I40E_PHY_TYPE_100BASE_TX:
1676                         ifmr->ifm_active |= IFM_100_TX;
1677                         break;
1678                 /* 1 G */
1679                 case I40E_PHY_TYPE_1000BASE_T:
1680                         ifmr->ifm_active |= IFM_1000_T;
1681                         break;
1682                 case I40E_PHY_TYPE_1000BASE_SX:
1683                         ifmr->ifm_active |= IFM_1000_SX;
1684                         break;
1685                 case I40E_PHY_TYPE_1000BASE_LX:
1686                         ifmr->ifm_active |= IFM_1000_LX;
1687                         break;
1688                 case I40E_PHY_TYPE_1000BASE_T_OPTICAL:
1689                         ifmr->ifm_active |= IFM_OTHER;
1690                         break;
1691                 /* 10 G */
1692                 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1693                         ifmr->ifm_active |= IFM_10G_TWINAX;
1694                         break;
1695                 case I40E_PHY_TYPE_10GBASE_SR:
1696                         ifmr->ifm_active |= IFM_10G_SR;
1697                         break;
1698                 case I40E_PHY_TYPE_10GBASE_LR:
1699                         ifmr->ifm_active |= IFM_10G_LR;
1700                         break;
1701                 case I40E_PHY_TYPE_10GBASE_T:
1702                         ifmr->ifm_active |= IFM_10G_T;
1703                         break;
1704                 case I40E_PHY_TYPE_XAUI:
1705                 case I40E_PHY_TYPE_XFI:
1706                 case I40E_PHY_TYPE_10GBASE_AOC:
1707                         ifmr->ifm_active |= IFM_OTHER;
1708                         break;
1709                 /* 40 G */
1710                 case I40E_PHY_TYPE_40GBASE_CR4:
1711                 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1712                         ifmr->ifm_active |= IFM_40G_CR4;
1713                         break;
1714                 case I40E_PHY_TYPE_40GBASE_SR4:
1715                         ifmr->ifm_active |= IFM_40G_SR4;
1716                         break;
1717                 case I40E_PHY_TYPE_40GBASE_LR4:
1718                         ifmr->ifm_active |= IFM_40G_LR4;
1719                         break;
1720                 case I40E_PHY_TYPE_XLAUI:
1721                         ifmr->ifm_active |= IFM_OTHER;
1722                         break;
1723 #ifndef IFM_ETH_XTYPE
1724                 case I40E_PHY_TYPE_1000BASE_KX:
1725                         ifmr->ifm_active |= IFM_1000_CX;
1726                         break;
1727                 case I40E_PHY_TYPE_SGMII:
1728                         ifmr->ifm_active |= IFM_OTHER;
1729                         break;
1730                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1731                 case I40E_PHY_TYPE_10GBASE_CR1:
1732                         ifmr->ifm_active |= IFM_10G_TWINAX;
1733                         break;
1734                 case I40E_PHY_TYPE_10GBASE_KX4:
1735                         ifmr->ifm_active |= IFM_10G_CX4;
1736                         break;
1737                 case I40E_PHY_TYPE_10GBASE_KR:
1738                         ifmr->ifm_active |= IFM_10G_SR;
1739                         break;
1740                 case I40E_PHY_TYPE_SFI:
1741                         ifmr->ifm_active |= IFM_OTHER;
1742                         break;
1743                 case I40E_PHY_TYPE_40GBASE_KR4:
1744                 case I40E_PHY_TYPE_XLPPI:
1745                 case I40E_PHY_TYPE_40GBASE_AOC:
1746                         ifmr->ifm_active |= IFM_40G_SR4;
1747                         break;
1748 #else
1749                 case I40E_PHY_TYPE_1000BASE_KX:
1750                         ifmr->ifm_active |= IFM_1000_KX;
1751                         break;
1752                 case I40E_PHY_TYPE_SGMII:
1753                         ifmr->ifm_active |= IFM_1000_SGMII;
1754                         break;
1755                 /* ERJ: What's the difference between these? */
1756                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1757                 case I40E_PHY_TYPE_10GBASE_CR1:
1758                         ifmr->ifm_active |= IFM_10G_CR1;
1759                         break;
1760                 case I40E_PHY_TYPE_10GBASE_KX4:
1761                         ifmr->ifm_active |= IFM_10G_KX4;
1762                         break;
1763                 case I40E_PHY_TYPE_10GBASE_KR:
1764                         ifmr->ifm_active |= IFM_10G_KR;
1765                         break;
1766                 case I40E_PHY_TYPE_SFI:
1767                         ifmr->ifm_active |= IFM_10G_SFI;
1768                         break;
1769                 /* Our single 20G media type */
1770                 case I40E_PHY_TYPE_20GBASE_KR2:
1771                         ifmr->ifm_active |= IFM_20G_KR2;
1772                         break;
1773                 case I40E_PHY_TYPE_40GBASE_KR4:
1774                         ifmr->ifm_active |= IFM_40G_KR4;
1775                         break;
1776                 case I40E_PHY_TYPE_XLPPI:
1777                 case I40E_PHY_TYPE_40GBASE_AOC:
1778                         ifmr->ifm_active |= IFM_40G_XLPPI;
1779                         break;
1780 #endif
1781                 /* Unknown to driver */
1782                 default:
1783                         ifmr->ifm_active |= IFM_UNKNOWN;
1784                         break;
1785         }
1786         /* Report flow control status as well */
1787         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1788                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1789         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1790                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1791
1792         IXL_PF_UNLOCK(pf);
1793
1794         return;
1795 }
1796
1797 /*
1798  * NOTE: Fortville does not support forcing media speeds. Instead,
1799  * use the set_advertise sysctl to set the speeds Fortville
1800  * will advertise or be allowed to operate at.
1801  */
1802 static int
1803 ixl_media_change(struct ifnet * ifp)
1804 {
1805         struct ixl_vsi *vsi = ifp->if_softc;
1806         struct ifmedia *ifm = &vsi->media;
1807
1808         INIT_DEBUGOUT("ixl_media_change: begin");
1809
1810         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1811                 return (EINVAL);
1812
1813         if_printf(ifp, "Media change is not supported.\n");
1814
1815         return (ENODEV);
1816 }
1817
1818
1819 #ifdef IXL_FDIR
1820 /*
1821 ** ATR: Application Targetted Receive - creates a filter
1822 **      based on TX flow info that will keep the receive
1823 **      portion of the flow on the same queue. Based on the
1824 **      implementation this is only available for TCP connections
1825 */
1826 void
1827 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1828 {
1829         struct ixl_vsi                  *vsi = que->vsi;
1830         struct tx_ring                  *txr = &que->txr;
1831         struct i40e_filter_program_desc *FDIR;
1832         u32                             ptype, dtype;
1833         int                             idx;
1834
1835         /* check if ATR is enabled and sample rate */
1836         if ((!ixl_enable_fdir) || (!txr->atr_rate))
1837                 return;
1838         /*
1839         ** We sample all TCP SYN/FIN packets,
1840         ** or at the selected sample rate
1841         */
1842         txr->atr_count++;
1843         if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1844             (txr->atr_count < txr->atr_rate))
1845                 return;
1846         txr->atr_count = 0;
1847
1848         /* Get a descriptor to use */
1849         idx = txr->next_avail;
1850         FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1851         if (++idx == que->num_desc)
1852                 idx = 0;
1853         txr->avail--;
1854         txr->next_avail = idx;
1855
1856         ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1857             I40E_TXD_FLTR_QW0_QINDEX_MASK;
1858
1859         ptype |= (etype == ETHERTYPE_IP) ?
1860             (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1861             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1862             (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1863             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1864
1865         ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1866
1867         dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1868
1869         /*
1870         ** We use the TCP TH_FIN as a trigger to remove
1871         ** the filter, otherwise its an update.
1872         */
1873         dtype |= (th->th_flags & TH_FIN) ?
1874             (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1875             I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1876             (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1877             I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1878
1879         dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1880             I40E_TXD_FLTR_QW1_DEST_SHIFT;
1881
1882         dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1883             I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1884
1885         FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1886         FDIR->dtype_cmd_cntindex = htole32(dtype);
1887         return;
1888 }
1889 #endif
1890
1891
1892 static void
1893 ixl_set_promisc(struct ixl_vsi *vsi)
1894 {
1895         struct ifnet    *ifp = vsi->ifp;
1896         struct i40e_hw  *hw = vsi->hw;
1897         int             err, mcnt = 0;
1898         bool            uni = FALSE, multi = FALSE;
1899
1900         if (ifp->if_flags & IFF_ALLMULTI)
1901                 multi = TRUE;
1902         else { /* Need to count the multicast addresses */
1903                 struct  ifmultiaddr *ifma;
1904                 if_maddr_rlock(ifp);
1905                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1906                         if (ifma->ifma_addr->sa_family != AF_LINK)
1907                                 continue;
1908                         if (mcnt == MAX_MULTICAST_ADDR)
1909                                 break;
1910                         mcnt++;
1911                 }
1912                 if_maddr_runlock(ifp);
1913         }
1914
1915         if (mcnt >= MAX_MULTICAST_ADDR)
1916                 multi = TRUE;
1917         if (ifp->if_flags & IFF_PROMISC)
1918                 uni = TRUE;
1919
1920         err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1921             vsi->seid, uni, NULL);
1922         err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1923             vsi->seid, multi, NULL);
1924         return;
1925 }
1926
1927 /*********************************************************************
1928  *      Filter Routines
1929  *
1930  *      Routines for multicast and vlan filter management.
1931  *
1932  *********************************************************************/
1933 static void
1934 ixl_add_multi(struct ixl_vsi *vsi)
1935 {
1936         struct  ifmultiaddr     *ifma;
1937         struct ifnet            *ifp = vsi->ifp;
1938         struct i40e_hw          *hw = vsi->hw;
1939         int                     mcnt = 0, flags;
1940
1941         IOCTL_DEBUGOUT("ixl_add_multi: begin");
1942
1943         if_maddr_rlock(ifp);
1944         /*
1945         ** First just get a count, to decide if we
1946         ** we simply use multicast promiscuous.
1947         */
1948         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1949                 if (ifma->ifma_addr->sa_family != AF_LINK)
1950                         continue;
1951                 mcnt++;
1952         }
1953         if_maddr_runlock(ifp);
1954
1955         if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1956                 /* delete existing MC filters */
1957                 ixl_del_hw_filters(vsi, mcnt);
1958                 i40e_aq_set_vsi_multicast_promiscuous(hw,
1959                     vsi->seid, TRUE, NULL);
1960                 return;
1961         }
1962
1963         mcnt = 0;
1964         if_maddr_rlock(ifp);
1965         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1966                 if (ifma->ifma_addr->sa_family != AF_LINK)
1967                         continue;
1968                 ixl_add_mc_filter(vsi,
1969                     (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1970                 mcnt++;
1971         }
1972         if_maddr_runlock(ifp);
1973         if (mcnt > 0) {
1974                 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1975                 ixl_add_hw_filters(vsi, flags, mcnt);
1976         }
1977
1978         IOCTL_DEBUGOUT("ixl_add_multi: end");
1979         return;
1980 }
1981
1982 static void
1983 ixl_del_multi(struct ixl_vsi *vsi)
1984 {
1985         struct ifnet            *ifp = vsi->ifp;
1986         struct ifmultiaddr      *ifma;
1987         struct ixl_mac_filter   *f;
1988         int                     mcnt = 0;
1989         bool            match = FALSE;
1990
1991         IOCTL_DEBUGOUT("ixl_del_multi: begin");
1992
1993         /* Search for removed multicast addresses */
1994         if_maddr_rlock(ifp);
1995         SLIST_FOREACH(f, &vsi->ftl, next) {
1996                 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1997                         match = FALSE;
1998                         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1999                                 if (ifma->ifma_addr->sa_family != AF_LINK)
2000                                         continue;
2001                                 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2002                                 if (cmp_etheraddr(f->macaddr, mc_addr)) {
2003                                         match = TRUE;
2004                                         break;
2005                                 }
2006                         }
2007                         if (match == FALSE) {
2008                                 f->flags |= IXL_FILTER_DEL;
2009                                 mcnt++;
2010                         }
2011                 }
2012         }
2013         if_maddr_runlock(ifp);
2014
2015         if (mcnt > 0)
2016                 ixl_del_hw_filters(vsi, mcnt);
2017 }
2018
2019
2020 /*********************************************************************
2021  *  Timer routine
2022  *
2023  *  This routine checks for link status,updates statistics,
2024  *  and runs the watchdog check.
2025  *
2026  *  Only runs when the driver is configured UP and RUNNING.
2027  *
2028  **********************************************************************/
2029
2030 static void
2031 ixl_local_timer(void *arg)
2032 {
2033         struct ixl_pf           *pf = arg;
2034         struct i40e_hw          *hw = &pf->hw;
2035         struct ixl_vsi          *vsi = &pf->vsi;
2036         struct ixl_queue        *que = vsi->queues;
2037         device_t                dev = pf->dev;
2038         int                     hung = 0;
2039         u32                     mask;
2040
2041         mtx_assert(&pf->pf_mtx, MA_OWNED);
2042
2043         /* Fire off the adminq task */
2044         taskqueue_enqueue(pf->tq, &pf->adminq);
2045
2046         /* Update stats */
2047         ixl_update_stats_counters(pf);
2048
2049         /*
2050         ** Check status of the queues
2051         */
2052         mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
2053                 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
2054
2055         for (int i = 0; i < vsi->num_queues; i++, que++) {
2056                 /* Any queues with outstanding work get a sw irq */
2057                 if (que->busy)
2058                         wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
2059                 /*
2060                 ** Each time txeof runs without cleaning, but there
2061                 ** are uncleaned descriptors it increments busy. If
2062                 ** we get to 5 we declare it hung.
2063                 */
2064                 if (que->busy == IXL_QUEUE_HUNG) {
2065                         ++hung;
2066                         /* Mark the queue as inactive */
2067                         vsi->active_queues &= ~((u64)1 << que->me);
2068                         continue;
2069                 } else {
2070                         /* Check if we've come back from hung */
2071                         if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
2072                                 vsi->active_queues |= ((u64)1 << que->me);
2073                 }
2074                 if (que->busy >= IXL_MAX_TX_BUSY) {
2075 #ifdef IXL_DEBUG
2076                         device_printf(dev,"Warning queue %d "
2077                             "appears to be hung!\n", i);
2078 #endif
2079                         que->busy = IXL_QUEUE_HUNG;
2080                         ++hung;
2081                 }
2082         }
2083         /* Only reinit if all queues show hung */
2084         if (hung == vsi->num_queues)
2085                 goto hung;
2086
2087         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
2088         return;
2089
2090 hung:
2091         device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
2092         ixl_init_locked(pf);
2093 }
2094
2095 /*
2096 ** Note: this routine updates the OS on the link state
2097 **      the real check of the hardware only happens with
2098 **      a link interrupt.
2099 */
2100 static void
2101 ixl_update_link_status(struct ixl_pf *pf)
2102 {
2103         struct ixl_vsi          *vsi = &pf->vsi;
2104         struct i40e_hw          *hw = &pf->hw;
2105         struct ifnet            *ifp = vsi->ifp;
2106         device_t                dev = pf->dev;
2107
2108         if (pf->link_up) {
2109                 if (vsi->link_active == FALSE) {
2110                         pf->fc = hw->fc.current_mode;
2111                         if (bootverbose) {
2112                                 device_printf(dev,"Link is up %d Gbps %s,"
2113                                     " Flow Control: %s\n",
2114                                     ((pf->link_speed ==
2115                                     I40E_LINK_SPEED_40GB)? 40:10),
2116                                     "Full Duplex", ixl_fc_string[pf->fc]);
2117                         }
2118                         vsi->link_active = TRUE;
2119                         /*
2120                         ** Warn user if link speed on NPAR enabled
2121                         ** partition is not at least 10GB
2122                         */
2123                         if (hw->func_caps.npar_enable &&
2124                            (hw->phy.link_info.link_speed ==
2125                            I40E_LINK_SPEED_1GB ||
2126                            hw->phy.link_info.link_speed ==
2127                            I40E_LINK_SPEED_100MB))
2128                                 device_printf(dev, "The partition detected"
2129                                     "link speed that is less than 10Gbps\n");
2130                         if_link_state_change(ifp, LINK_STATE_UP);
2131                 }
2132         } else { /* Link down */
2133                 if (vsi->link_active == TRUE) {
2134                         if (bootverbose)
2135                                 device_printf(dev, "Link is Down\n");
2136                         if_link_state_change(ifp, LINK_STATE_DOWN);
2137                         vsi->link_active = FALSE;
2138                 }
2139         }
2140
2141         return;
2142 }
2143
2144 static void
2145 ixl_stop(struct ixl_pf *pf)
2146 {
2147         IXL_PF_LOCK(pf);
2148         ixl_stop_locked(pf);
2149         IXL_PF_UNLOCK(pf);
2150
2151         ixl_teardown_queue_msix(&pf->vsi);
2152         ixl_free_queue_tqs(&pf->vsi);
2153 }
2154
2155 /*********************************************************************
2156  *
2157  *  This routine disables all traffic on the adapter by issuing a
2158  *  global reset on the MAC and deallocates TX/RX buffers.
2159  *
2160  **********************************************************************/
2161
2162 static void
2163 ixl_stop_locked(struct ixl_pf *pf)
2164 {
2165         struct ixl_vsi  *vsi = &pf->vsi;
2166         struct ifnet    *ifp = vsi->ifp;
2167
2168         INIT_DEBUGOUT("ixl_stop: begin\n");
2169
2170         IXL_PF_LOCK_ASSERT(pf);
2171
2172         /* Stop the local timer */
2173         callout_stop(&pf->timer);
2174
2175         ixl_disable_rings_intr(vsi);
2176         ixl_disable_rings(vsi);
2177
2178         /* Tell the stack that the interface is no longer active */
2179         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING);
2180 }
2181
2182
2183 /*********************************************************************
2184  *
2185  *  Setup MSIX Interrupt resources and handlers for the VSI
2186  *
2187  **********************************************************************/
2188 static int
2189 ixl_assign_vsi_legacy(struct ixl_pf *pf)
2190 {
2191         device_t        dev = pf->dev;
2192         struct          ixl_vsi *vsi = &pf->vsi;
2193         struct          ixl_queue *que = vsi->queues;
2194         int             error, rid = 0;
2195
2196         if (pf->msix == 1)
2197                 rid = 1;
2198         pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2199             &rid, RF_SHAREABLE | RF_ACTIVE);
2200         if (pf->res == NULL) {
2201                 device_printf(dev, "Unable to allocate"
2202                     " bus resource: vsi legacy/msi interrupt\n");
2203                 return (ENXIO);
2204         }
2205
2206         /* Set the handler function */
2207         error = bus_setup_intr(dev, pf->res,
2208             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2209             ixl_intr, pf, &pf->tag);
2210         if (error) {
2211                 pf->res = NULL;
2212                 device_printf(dev, "Failed to register legacy/msi handler\n");
2213                 return (error);
2214         }
2215         bus_describe_intr(dev, pf->res, pf->tag, "irq0");
2216         TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2217         TASK_INIT(&que->task, 0, ixl_handle_que, que);
2218         que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2219             taskqueue_thread_enqueue, &que->tq);
2220         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2221             device_get_nameunit(dev));
2222         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2223
2224         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2225             taskqueue_thread_enqueue, &pf->tq);
2226         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2227             device_get_nameunit(dev));
2228
2229         return (0);
2230 }
2231
2232 static int
2233 ixl_setup_adminq_tq(struct ixl_pf *pf)
2234 {
2235         device_t dev = pf->dev;
2236         int error = 0;
2237
2238         /* Tasklet for Admin Queue interrupts */
2239         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2240 #ifdef PCI_IOV
2241         /* VFLR Tasklet */
2242         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2243 #endif
2244         /* Create and start Admin Queue taskqueue */
2245         pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT,
2246             taskqueue_thread_enqueue, &pf->tq);
2247         if (!pf->tq) {
2248                 device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n");
2249                 return (ENOMEM);
2250         }
2251         error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq",
2252             device_get_nameunit(dev));
2253         if (error) {
2254                 device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n",
2255                     error);
2256                 taskqueue_free(pf->tq);
2257                 return (error);
2258         }
2259         return (0);
2260 }
2261
2262 static int
2263 ixl_setup_queue_tqs(struct ixl_vsi *vsi)
2264 {
2265         struct ixl_queue *que = vsi->queues;
2266         device_t dev = vsi->dev;
2267
2268         /* Create queue tasks and start queue taskqueues */
2269         for (int i = 0; i < vsi->num_queues; i++, que++) {
2270                 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2271                 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2272                 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2273                     taskqueue_thread_enqueue, &que->tq);
2274 #ifdef RSS
2275                 CPU_SETOF(cpu_id, &cpu_mask);
2276                 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2277                     &cpu_mask, "%s (bucket %d)",
2278                     device_get_nameunit(dev), cpu_id);
2279 #else
2280                 taskqueue_start_threads(&que->tq, 1, PI_NET,
2281                     "%s (que %d)", device_get_nameunit(dev), que->me);
2282 #endif
2283         }
2284
2285         return (0);
2286 }
2287
2288 static void
2289 ixl_free_adminq_tq(struct ixl_pf *pf)
2290 {
2291         if (pf->tq) {
2292                 taskqueue_free(pf->tq);
2293                 pf->tq = NULL;
2294         }
2295 }
2296
2297 static void
2298 ixl_free_queue_tqs(struct ixl_vsi *vsi)
2299 {
2300         struct ixl_queue *que = vsi->queues;
2301
2302         for (int i = 0; i < vsi->num_queues; i++, que++) {
2303                 if (que->tq) {
2304                         taskqueue_free(que->tq);
2305                         que->tq = NULL;
2306                 }
2307         }
2308 }
2309
2310 static int
2311 ixl_setup_adminq_msix(struct ixl_pf *pf)
2312 {
2313         device_t dev = pf->dev;
2314         int rid, error = 0;
2315
2316         /* Admin IRQ rid is 1, vector is 0 */
2317         rid = 1;
2318         /* Get interrupt resource from bus */
2319         pf->res = bus_alloc_resource_any(dev,
2320             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2321         if (!pf->res) {
2322                 device_printf(dev, "bus_alloc_resource_any() for Admin Queue"
2323                     " interrupt failed [rid=%d]\n", rid);
2324                 return (ENXIO);
2325         }
2326         /* Then associate interrupt with handler */
2327         error = bus_setup_intr(dev, pf->res,
2328             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2329             ixl_msix_adminq, pf, &pf->tag);
2330         if (error) {
2331                 pf->res = NULL;
2332                 device_printf(dev, "bus_setup_intr() for Admin Queue"
2333                     " interrupt handler failed, error %d\n", error);
2334                 return (ENXIO);
2335         }
2336         error = bus_describe_intr(dev, pf->res, pf->tag, "aq");
2337         if (error) {
2338                 /* Probably non-fatal? */
2339                 device_printf(dev, "bus_describe_intr() for Admin Queue"
2340                     " interrupt name failed, error %d\n", error);
2341         }
2342         pf->admvec = 0;
2343
2344         return (0);
2345 }
2346
2347 /*
2348  * Allocate interrupt resources from bus and associate an interrupt handler
2349  * to those for the VSI's queues.
2350  */
2351 static int
2352 ixl_setup_queue_msix(struct ixl_vsi *vsi)
2353 {
2354         device_t        dev = vsi->dev;
2355         struct          ixl_queue *que = vsi->queues;
2356         struct          tx_ring  *txr;
2357         int             error, rid, vector = 1;
2358 #ifdef  RSS
2359         cpuset_t cpu_mask;
2360 #endif
2361
2362         /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */
2363         for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2364                 int cpu_id = i;
2365                 rid = vector + 1;
2366                 txr = &que->txr;
2367                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2368                     RF_SHAREABLE | RF_ACTIVE);
2369                 if (!que->res) {
2370                         device_printf(dev, "bus_alloc_resource_any() for"
2371                             " Queue %d interrupt failed [rid=%d]\n",
2372                             que->me, rid);
2373                         return (ENXIO);
2374                 }
2375                 /* Set the handler function */
2376                 error = bus_setup_intr(dev, que->res,
2377                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2378                     ixl_msix_que, que, &que->tag);
2379                 if (error) {
2380                         device_printf(dev, "bus_setup_intr() for Queue %d"
2381                             " interrupt handler failed, error %d\n",
2382                             que->me, error);
2383                         // TODO: Check for error from this?
2384                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2385                         return (error);
2386                 }
2387                 error = bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2388                 if (error) {
2389                         device_printf(dev, "bus_describe_intr() for Queue %d"
2390                             " interrupt name failed, error %d\n",
2391                             que->me, error);
2392                 }
2393                 /* Bind the vector to a CPU */
2394 #ifdef RSS
2395                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2396 #endif
2397                 error = bus_bind_intr(dev, que->res, cpu_id);
2398                 if (error) {
2399                         device_printf(dev, "bus_bind_intr() for Queue %d"
2400                             " to CPU %d failed, error %d\n",
2401                             que->me, cpu_id, error);
2402                 }
2403                 que->msix = vector;
2404         }
2405
2406         return (0);
2407 }
2408
2409
2410 /*
2411  * Allocate MSI/X vectors
2412  */
2413 static int
2414 ixl_init_msix(struct ixl_pf *pf)
2415 {
2416         device_t dev = pf->dev;
2417         int rid, want, vectors, queues, available;
2418
2419         /* Override by tuneable */
2420         if (ixl_enable_msix == 0)
2421                 goto no_msix;
2422
2423         /*
2424         ** When used in a virtualized environment
2425         ** PCI BUSMASTER capability may not be set
2426         ** so explicity set it here and rewrite
2427         ** the ENABLE in the MSIX control register
2428         ** at this point to cause the host to
2429         ** successfully initialize us.
2430         */
2431         {
2432                 u16 pci_cmd_word;
2433                 int msix_ctrl;
2434                 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2435                 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2436                 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2437                 pci_find_cap(dev, PCIY_MSIX, &rid);
2438                 rid += PCIR_MSIX_CTRL;
2439                 msix_ctrl = pci_read_config(dev, rid, 2);
2440                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2441                 pci_write_config(dev, rid, msix_ctrl, 2);
2442         }
2443
2444         /* First try MSI/X */
2445         rid = PCIR_BAR(IXL_BAR);
2446         pf->msix_mem = bus_alloc_resource_any(dev,
2447             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2448         if (!pf->msix_mem) {
2449                 /* May not be enabled */
2450                 device_printf(pf->dev,
2451                     "Unable to map MSIX table\n");
2452                 goto no_msix;
2453         }
2454
2455         available = pci_msix_count(dev);
2456         if (available == 0) { /* system has msix disabled */
2457                 bus_release_resource(dev, SYS_RES_MEMORY,
2458                     rid, pf->msix_mem);
2459                 pf->msix_mem = NULL;
2460                 goto no_msix;
2461         }
2462
2463         /* Figure out a reasonable auto config value */
2464         queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2465
2466         /* Override with tunable value if tunable is less than autoconfig count */
2467         if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2468                 queues = ixl_max_queues;
2469         else if ((ixl_max_queues != 0) && (ixl_max_queues > queues))
2470                 device_printf(dev, "ixl_max_queues > # of cpus, using "
2471                     "autoconfig amount...\n");
2472         /* Or limit maximum auto-configured queues to 8 */
2473         else if ((ixl_max_queues == 0) && (queues > 8))
2474                 queues = 8;
2475
2476 #ifdef  RSS
2477         /* If we're doing RSS, clamp at the number of RSS buckets */
2478         if (queues > rss_getnumbuckets())
2479                 queues = rss_getnumbuckets();
2480 #endif
2481
2482         /*
2483         ** Want one vector (RX/TX pair) per queue
2484         ** plus an additional for the admin queue.
2485         */
2486         want = queues + 1;
2487         if (want <= available)  /* Have enough */
2488                 vectors = want;
2489         else {
2490                 device_printf(pf->dev,
2491                     "MSIX Configuration Problem, "
2492                     "%d vectors available but %d wanted!\n",
2493                     available, want);
2494                 return (0); /* Will go to Legacy setup */
2495         }
2496
2497         if (pci_alloc_msix(dev, &vectors) == 0) {
2498                 device_printf(pf->dev,
2499                     "Using MSIX interrupts with %d vectors\n", vectors);
2500                 pf->msix = vectors;
2501                 pf->vsi.num_queues = queues;
2502 #ifdef RSS
2503                 /*
2504                  * If we're doing RSS, the number of queues needs to
2505                  * match the number of RSS buckets that are configured.
2506                  *
2507                  * + If there's more queues than RSS buckets, we'll end
2508                  *   up with queues that get no traffic.
2509                  *
2510                  * + If there's more RSS buckets than queues, we'll end
2511                  *   up having multiple RSS buckets map to the same queue,
2512                  *   so there'll be some contention.
2513                  */
2514                 if (queues != rss_getnumbuckets()) {
2515                         device_printf(dev,
2516                             "%s: queues (%d) != RSS buckets (%d)"
2517                             "; performance will be impacted.\n",
2518                             __func__, queues, rss_getnumbuckets());
2519                 }
2520 #endif
2521                 return (vectors);
2522         }
2523 no_msix:
2524         vectors = pci_msi_count(dev);
2525         pf->vsi.num_queues = 1;
2526         ixl_max_queues = 1;
2527         ixl_enable_msix = 0;
2528         if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2529                 device_printf(pf->dev, "Using an MSI interrupt\n");
2530         else {
2531                 vectors = 0;
2532                 device_printf(pf->dev, "Using a Legacy interrupt\n");
2533         }
2534         return (vectors);
2535 }
2536
2537 /*
2538  * Configure admin queue/misc interrupt cause registers in hardware.
2539  */
2540 static void
2541 ixl_configure_intr0_msix(struct ixl_pf *pf)
2542 {
2543         struct i40e_hw *hw = &pf->hw;
2544         u32 reg;
2545
2546         /* First set up the adminq - vector 0 */
2547         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2548         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2549
2550         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2551             I40E_PFINT_ICR0_ENA_GRST_MASK |
2552             I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
2553             I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2554             I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2555             I40E_PFINT_ICR0_ENA_VFLR_MASK |
2556             I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2557         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2558
2559         /*
2560          * 0x7FF is the end of the queue list.
2561          * This means we won't use MSI-X vector 0 for a queue interrupt
2562          * in MSIX mode.
2563          */
2564         wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2565         /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */
2566         wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E);
2567
2568         wr32(hw, I40E_PFINT_DYN_CTL0,
2569             I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2570             I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2571
2572         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2573 }
2574
2575 /*
2576  * Configure queue interrupt cause registers in hardware.
2577  */
2578 static void
2579 ixl_configure_queue_intr_msix(struct ixl_pf *pf)
2580 {
2581         struct i40e_hw  *hw = &pf->hw;
2582         struct ixl_vsi *vsi = &pf->vsi;
2583         u32             reg;
2584         u16             vector = 1;
2585
2586         for (int i = 0; i < vsi->num_queues; i++, vector++) {
2587                 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
2588                 /* First queue type is RX / 0 */
2589                 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2590
2591                 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2592                 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2593                 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2594                 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2595                 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2596                 wr32(hw, I40E_QINT_RQCTL(i), reg);
2597
2598                 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2599                 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2600                 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2601                 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2602                 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2603                 wr32(hw, I40E_QINT_TQCTL(i), reg);
2604         }
2605 }
2606
2607 /*
2608  * Configure for MSI single vector operation
2609  */
2610 static void
2611 ixl_configure_legacy(struct ixl_pf *pf)
2612 {
2613         struct i40e_hw  *hw = &pf->hw;
2614         u32             reg;
2615
2616         wr32(hw, I40E_PFINT_ITR0(0), 0);
2617         wr32(hw, I40E_PFINT_ITR0(1), 0);
2618
2619         /* Setup "other" causes */
2620         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2621             | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2622             | I40E_PFINT_ICR0_ENA_GRST_MASK
2623             | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2624             | I40E_PFINT_ICR0_ENA_GPIO_MASK
2625             | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2626             | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2627             | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2628             | I40E_PFINT_ICR0_ENA_VFLR_MASK
2629             | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2630             ;
2631         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2632
2633         /* SW_ITR_IDX = 0, but don't change INTENA */
2634         wr32(hw, I40E_PFINT_DYN_CTL0,
2635             I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2636             I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2637         /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2638         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2639
2640         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2641         wr32(hw, I40E_PFINT_LNKLST0, 0);
2642
2643         /* Associate the queue pair to the vector and enable the q int */
2644         reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2645             | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2646             | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2647         wr32(hw, I40E_QINT_RQCTL(0), reg);
2648
2649         reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2650             | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2651             | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2652         wr32(hw, I40E_QINT_TQCTL(0), reg);
2653 }
2654
2655
2656 /*
2657  * Get initial ITR values from tunable values.
2658  */
2659 static void
2660 ixl_configure_itr(struct ixl_pf *pf)
2661 {
2662         struct i40e_hw          *hw = &pf->hw;
2663         struct ixl_vsi          *vsi = &pf->vsi;
2664         struct ixl_queue        *que = vsi->queues;
2665
2666         vsi->rx_itr_setting = ixl_rx_itr;
2667         vsi->tx_itr_setting = ixl_tx_itr;
2668
2669         for (int i = 0; i < vsi->num_queues; i++, que++) {
2670                 struct tx_ring  *txr = &que->txr;
2671                 struct rx_ring  *rxr = &que->rxr;
2672
2673                 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2674                     vsi->rx_itr_setting);
2675                 rxr->itr = vsi->rx_itr_setting;
2676                 rxr->latency = IXL_AVE_LATENCY;
2677
2678                 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2679                     vsi->tx_itr_setting);
2680                 txr->itr = vsi->tx_itr_setting;
2681                 txr->latency = IXL_AVE_LATENCY;
2682         }
2683 }
2684
2685
2686 static int
2687 ixl_allocate_pci_resources(struct ixl_pf *pf)
2688 {
2689         int             rid;
2690         device_t        dev = pf->dev;
2691
2692         rid = PCIR_BAR(0);
2693         pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2694             &rid, RF_ACTIVE);
2695
2696         if (!(pf->pci_mem)) {
2697                 device_printf(dev, "Unable to allocate bus resource: PCI memory\n");
2698                 return (ENXIO);
2699         }
2700
2701         pf->osdep.mem_bus_space_tag =
2702                 rman_get_bustag(pf->pci_mem);
2703         pf->osdep.mem_bus_space_handle =
2704                 rman_get_bushandle(pf->pci_mem);
2705         pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2706         pf->osdep.flush_reg = I40E_GLGEN_STAT;
2707         pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2708
2709         pf->hw.back = &pf->osdep;
2710
2711         /*
2712         ** Now setup MSI or MSI/X, should
2713         ** return us the number of supported
2714         ** vectors. (Will be 1 for MSI)
2715         */
2716         pf->msix = ixl_init_msix(pf);
2717         return (0);
2718 }
2719
2720 /*
2721  * Teardown and release the admin queue/misc vector
2722  * interrupt.
2723  */
2724 static int
2725 ixl_teardown_adminq_msix(struct ixl_pf *pf)
2726 {
2727         device_t                dev = pf->dev;
2728         int                     rid;
2729
2730         if (pf->admvec) /* we are doing MSIX */
2731                 rid = pf->admvec + 1;
2732         else
2733                 (pf->msix != 0) ? (rid = 1):(rid = 0);
2734
2735         // TODO: Check for errors from bus_teardown_intr
2736         // TODO: Check for errors from bus_release_resource
2737         if (pf->tag != NULL) {
2738                 bus_teardown_intr(dev, pf->res, pf->tag);
2739                 pf->tag = NULL;
2740         }
2741         if (pf->res != NULL) {
2742                 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2743                 pf->res = NULL;
2744         }
2745
2746         return (0);
2747 }
2748
2749 static int
2750 ixl_teardown_queue_msix(struct ixl_vsi *vsi)
2751 {
2752         struct ixl_queue        *que = vsi->queues;
2753         device_t                dev = vsi->dev;
2754         int                     rid, error = 0;
2755
2756         /* We may get here before stations are setup */
2757         if ((!ixl_enable_msix) || (que == NULL))
2758                 return (0);
2759
2760         /* Release all MSIX queue resources */
2761         for (int i = 0; i < vsi->num_queues; i++, que++) {
2762                 rid = que->msix + 1;
2763                 if (que->tag != NULL) {
2764                         error = bus_teardown_intr(dev, que->res, que->tag);
2765                         if (error) {
2766                                 device_printf(dev, "bus_teardown_intr() for"
2767                                     " Queue %d interrupt failed\n",
2768                                     que->me);
2769                                 // return (ENXIO);
2770                         }
2771                         que->tag = NULL;
2772                 }
2773                 if (que->res != NULL) {
2774                         error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2775                         if (error) {
2776                                 device_printf(dev, "bus_release_resource() for"
2777                                     " Queue %d interrupt failed [rid=%d]\n",
2778                                     que->me, rid);
2779                                 // return (ENXIO);
2780                         }
2781                         que->res = NULL;
2782                 }
2783         }
2784
2785         return (0);
2786 }
2787
2788 static void
2789 ixl_free_pci_resources(struct ixl_pf *pf)
2790 {
2791         device_t                dev = pf->dev;
2792         int                     memrid;
2793
2794         ixl_teardown_queue_msix(&pf->vsi);
2795         ixl_teardown_adminq_msix(pf);
2796
2797         if (pf->msix)
2798                 pci_release_msi(dev);
2799
2800         memrid = PCIR_BAR(IXL_BAR);
2801
2802         if (pf->msix_mem != NULL)
2803                 bus_release_resource(dev, SYS_RES_MEMORY,
2804                     memrid, pf->msix_mem);
2805
2806         if (pf->pci_mem != NULL)
2807                 bus_release_resource(dev, SYS_RES_MEMORY,
2808                     PCIR_BAR(0), pf->pci_mem);
2809
2810         return;
2811 }
2812
2813 static void
2814 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2815 {
2816         /* Display supported media types */
2817         if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2818                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2819
2820         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2821                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2822         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2823                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2824         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2825                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2826
2827         if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2828             phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2829             phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2830                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2831
2832         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2833                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2834         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2835                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2836         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2837                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2838
2839         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2840             phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2841             phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2842             phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2843             phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2844                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2845         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2846                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2847         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2848                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2849
2850 #ifndef IFM_ETH_XTYPE
2851         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2852                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2853
2854         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2855             phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2856             phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2857             phy_type & (1 << I40E_PHY_TYPE_SFI))
2858                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2859         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2860                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2861         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2862                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2863
2864         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2865                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2866         if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2867                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2868 #else
2869         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2870                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2871
2872         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2873             || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2874                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2875         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2876                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2877         if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2878                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2879         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2880                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2881         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2882                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2883
2884         if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2885                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2886
2887         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2888                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2889         if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2890                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2891 #endif
2892 }
2893
2894 /*********************************************************************
2895  *
2896  *  Setup networking device structure and register an interface.
2897  *
2898  **********************************************************************/
2899 static int
2900 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2901 {
2902         struct ifnet            *ifp;
2903         struct i40e_hw          *hw = vsi->hw;
2904         struct ixl_queue        *que = vsi->queues;
2905         struct i40e_aq_get_phy_abilities_resp abilities;
2906         enum i40e_status_code aq_error = 0;
2907
2908         INIT_DEBUGOUT("ixl_setup_interface: begin");
2909
2910         ifp = vsi->ifp = if_alloc(IFT_ETHER);
2911         if (ifp == NULL) {
2912                 device_printf(dev, "can not allocate ifnet structure\n");
2913                 return (-1);
2914         }
2915         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2916         ifp->if_mtu = ETHERMTU;
2917         ifp->if_baudrate = IF_Gbps(40);
2918         ifp->if_init = ixl_init;
2919         ifp->if_softc = vsi;
2920         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2921         ifp->if_ioctl = ixl_ioctl;
2922
2923 #if __FreeBSD_version >= 1100036
2924         if_setgetcounterfn(ifp, ixl_get_counter);
2925 #endif
2926
2927         ifp->if_transmit = ixl_mq_start;
2928
2929         ifp->if_qflush = ixl_qflush;
2930
2931         ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2932
2933         vsi->max_frame_size =
2934             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2935             + ETHER_VLAN_ENCAP_LEN;
2936
2937         /*
2938          * Tell the upper layer(s) we support long frames.
2939          */
2940         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2941
2942         ifp->if_capabilities |= IFCAP_HWCSUM;
2943         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2944         ifp->if_capabilities |= IFCAP_TSO;
2945         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2946         ifp->if_capabilities |= IFCAP_LRO;
2947
2948         /* VLAN capabilties */
2949         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2950                              |  IFCAP_VLAN_HWTSO
2951                              |  IFCAP_VLAN_MTU
2952                              |  IFCAP_VLAN_HWCSUM;
2953         ifp->if_capenable = ifp->if_capabilities;
2954
2955         /*
2956         ** Don't turn this on by default, if vlans are
2957         ** created on another pseudo device (eg. lagg)
2958         ** then vlan events are not passed thru, breaking
2959         ** operation, but with HW FILTER off it works. If
2960         ** using vlans directly on the ixl driver you can
2961         ** enable this and get full hardware tag filtering.
2962         */
2963         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2964
2965         /*
2966          * Specify the media types supported by this adapter and register
2967          * callbacks to update media and link information
2968          */
2969         ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2970                      ixl_media_status);
2971
2972         aq_error = i40e_aq_get_phy_capabilities(hw,
2973             FALSE, TRUE, &abilities, NULL);
2974         /* May need delay to detect fiber correctly */
2975         if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2976                 i40e_msec_delay(200);
2977                 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2978                     TRUE, &abilities, NULL);
2979         }
2980         if (aq_error) {
2981                 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2982                         device_printf(dev, "Unknown PHY type detected!\n");
2983                 else
2984                         device_printf(dev,
2985                             "Error getting supported media types, err %d,"
2986                             " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2987                 return (0);
2988         }
2989
2990         ixl_add_ifmedia(vsi, abilities.phy_type);
2991
2992         /* Use autoselect media by default */
2993         ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2994         ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2995
2996         ether_ifattach(ifp, hw->mac.addr);
2997
2998         return (0);
2999 }
3000
3001 /*
3002 ** Run when the Admin Queue gets a link state change interrupt.
3003 */
3004 static void
3005 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
3006 {
3007         struct i40e_hw  *hw = &pf->hw;
3008         device_t dev = pf->dev;
3009         struct i40e_aqc_get_link_status *status =
3010             (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
3011
3012         /* Request link status from adapter */
3013         hw->phy.get_link_info = TRUE;
3014         i40e_get_link_status(hw, &pf->link_up);
3015
3016         /* Print out message if an unqualified module is found */
3017         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
3018             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
3019             (!(status->link_info & I40E_AQ_LINK_UP)))
3020                 device_printf(dev, "Link failed because "
3021                     "an unqualified module was detected!\n");
3022
3023         /* Update OS link info */
3024         ixl_update_link_status(pf);
3025 }
3026
3027 /*********************************************************************
3028  *
3029  *  Get Firmware Switch configuration
3030  *      - this will need to be more robust when more complex
3031  *        switch configurations are enabled.
3032  *
3033  **********************************************************************/
3034 static int
3035 ixl_switch_config(struct ixl_pf *pf)
3036 {
3037         struct i40e_hw  *hw = &pf->hw;
3038         struct ixl_vsi  *vsi = &pf->vsi;
3039         device_t        dev = vsi->dev;
3040         struct i40e_aqc_get_switch_config_resp *sw_config;
3041         u8      aq_buf[I40E_AQ_LARGE_BUF];
3042         int     ret;
3043         u16     next = 0;
3044
3045         memset(&aq_buf, 0, sizeof(aq_buf));
3046         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
3047         ret = i40e_aq_get_switch_config(hw, sw_config,
3048             sizeof(aq_buf), &next, NULL);
3049         if (ret) {
3050                 device_printf(dev, "aq_get_switch_config() failed, error %d,"
3051                     " aq_error %d\n", ret, pf->hw.aq.asq_last_status);
3052                 return (ret);
3053         }
3054 #ifdef IXL_DEBUG
3055         device_printf(dev,
3056             "Switch config: header reported: %d in structure, %d total\n",
3057             sw_config->header.num_reported, sw_config->header.num_total);
3058         for (int i = 0; i < sw_config->header.num_reported; i++) {
3059                 device_printf(dev,
3060                     "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
3061                     sw_config->element[i].element_type,
3062                     sw_config->element[i].seid,
3063                     sw_config->element[i].uplink_seid,
3064                     sw_config->element[i].downlink_seid);
3065         }
3066 #endif
3067         /* Simplified due to a single VSI at the moment */
3068         vsi->uplink_seid = sw_config->element[0].uplink_seid;
3069         vsi->downlink_seid = sw_config->element[0].downlink_seid;
3070         vsi->seid = sw_config->element[0].seid;
3071         return (ret);
3072 }
3073
3074 /*********************************************************************
3075  *
3076  *  Initialize the VSI:  this handles contexts, which means things
3077  *                       like the number of descriptors, buffer size,
3078  *                       plus we init the rings thru this function.
3079  *
3080  **********************************************************************/
3081 static int
3082 ixl_initialize_vsi(struct ixl_vsi *vsi)
3083 {
3084         struct ixl_pf           *pf = vsi->back;
3085         struct ixl_queue        *que = vsi->queues;
3086         device_t                dev = vsi->dev;
3087         struct i40e_hw          *hw = vsi->hw;
3088         struct i40e_vsi_context ctxt;
3089         int                     err = 0;
3090
3091         memset(&ctxt, 0, sizeof(ctxt));
3092         ctxt.seid = vsi->seid;
3093         if (pf->veb_seid != 0)
3094                 ctxt.uplink_seid = pf->veb_seid;
3095         ctxt.pf_num = hw->pf_id;
3096         err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
3097         if (err) {
3098                 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d"
3099                     " aq_error %d\n", err, hw->aq.asq_last_status);
3100                 return (err);
3101         }
3102 #ifdef IXL_DEBUG
3103         device_printf(dev, "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
3104             "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
3105             "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
3106             ctxt.uplink_seid, ctxt.vsi_number,
3107             ctxt.vsis_allocated, ctxt.vsis_unallocated,
3108             ctxt.flags, ctxt.pf_num, ctxt.vf_num,
3109             ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
3110 #endif
3111         /*
3112         ** Set the queue and traffic class bits
3113         **  - when multiple traffic classes are supported
3114         **    this will need to be more robust.
3115         */
3116         ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
3117         ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
3118         /* In contig mode, que_mapping[0] is first queue index used by this VSI */
3119         ctxt.info.queue_mapping[0] = 0;
3120         /*
3121          * This VSI will only use traffic class 0; start traffic class 0's
3122          * queue allocation at queue 0, and assign it 64 (2^6) queues (though
3123          * the driver may not use all of them).
3124          */
3125         ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
3126             & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
3127             ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
3128             & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
3129
3130         /* Set VLAN receive stripping mode */
3131         ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
3132         ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
3133         if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
3134                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3135         else
3136                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3137
3138         /* Keep copy of VSI info in VSI for statistic counters */
3139         memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
3140
3141         /* Reset VSI statistics */
3142         ixl_vsi_reset_stats(vsi);
3143         vsi->hw_filters_add = 0;
3144         vsi->hw_filters_del = 0;
3145
3146         ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
3147
3148         err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3149         if (err) {
3150                 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d, aq_error %d\n",
3151                    err, hw->aq.asq_last_status);
3152                 return (err);
3153         }
3154
3155         for (int i = 0; i < vsi->num_queues; i++, que++) {
3156                 struct tx_ring          *txr = &que->txr;
3157                 struct rx_ring          *rxr = &que->rxr;
3158                 struct i40e_hmc_obj_txq tctx;
3159                 struct i40e_hmc_obj_rxq rctx;
3160                 u32                     txctl;
3161                 u16                     size;
3162
3163                 /* Setup the HMC TX Context  */
3164                 size = que->num_desc * sizeof(struct i40e_tx_desc);
3165                 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
3166                 tctx.new_context = 1;
3167                 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
3168                 tctx.qlen = que->num_desc;
3169                 tctx.fc_ena = 0;
3170                 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
3171                 /* Enable HEAD writeback */
3172                 tctx.head_wb_ena = 1;
3173                 tctx.head_wb_addr = txr->dma.pa +
3174                     (que->num_desc * sizeof(struct i40e_tx_desc));
3175                 tctx.rdylist_act = 0;
3176                 err = i40e_clear_lan_tx_queue_context(hw, i);
3177                 if (err) {
3178                         device_printf(dev, "Unable to clear TX context\n");
3179                         break;
3180                 }
3181                 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
3182                 if (err) {
3183                         device_printf(dev, "Unable to set TX context\n");
3184                         break;
3185                 }
3186                 /* Associate the ring with this PF */
3187                 txctl = I40E_QTX_CTL_PF_QUEUE;
3188                 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3189                     I40E_QTX_CTL_PF_INDX_MASK);
3190                 wr32(hw, I40E_QTX_CTL(i), txctl);
3191                 ixl_flush(hw);
3192
3193                 /* Do ring (re)init */
3194                 ixl_init_tx_ring(que);
3195
3196                 /* Next setup the HMC RX Context  */
3197                 if (vsi->max_frame_size <= MCLBYTES)
3198                         rxr->mbuf_sz = MCLBYTES;
3199                 else
3200                         rxr->mbuf_sz = MJUMPAGESIZE;
3201
3202                 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
3203
3204                 /* Set up an RX context for the HMC */
3205                 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
3206                 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
3207                 /* ignore header split for now */
3208                 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
3209                 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
3210                     vsi->max_frame_size : max_rxmax;
3211                 rctx.dtype = 0;
3212                 rctx.dsize = 1; /* do 32byte descriptors */
3213                 rctx.hsplit_0 = 0;  /* no HDR split initially */
3214                 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
3215                 rctx.qlen = que->num_desc;
3216                 rctx.tphrdesc_ena = 1;
3217                 rctx.tphwdesc_ena = 1;
3218                 rctx.tphdata_ena = 0;
3219                 rctx.tphhead_ena = 0;
3220                 rctx.lrxqthresh = 2;
3221                 rctx.crcstrip = 1;
3222                 rctx.l2tsel = 1;
3223                 rctx.showiv = 1;
3224                 rctx.fc_ena = 0;
3225                 rctx.prefena = 1;
3226
3227                 err = i40e_clear_lan_rx_queue_context(hw, i);
3228                 if (err) {
3229                         device_printf(dev,
3230                             "Unable to clear RX context %d\n", i);
3231                         break;
3232                 }
3233                 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
3234                 if (err) {
3235                         device_printf(dev, "Unable to set RX context %d\n", i);
3236                         break;
3237                 }
3238                 err = ixl_init_rx_ring(que);
3239                 if (err) {
3240                         device_printf(dev, "Fail in init_rx_ring %d\n", i);
3241                         break;
3242                 }
3243 #ifdef DEV_NETMAP
3244                 /* preserve queue */
3245                 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
3246                         struct netmap_adapter *na = NA(vsi->ifp);
3247                         struct netmap_kring *kring = &na->rx_rings[i];
3248                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
3249                         wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
3250                 } else
3251 #endif /* DEV_NETMAP */
3252                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
3253         }
3254         return (err);
3255 }
3256
3257
3258 /*********************************************************************
3259  *
3260  *  Free all VSI structs.
3261  *
3262  **********************************************************************/
3263 void
3264 ixl_free_vsi(struct ixl_vsi *vsi)
3265 {
3266         struct ixl_pf           *pf = (struct ixl_pf *)vsi->back;
3267         struct ixl_queue        *que = vsi->queues;
3268
3269         /* Free station queues */
3270         if (!vsi->queues)
3271                 goto free_filters;
3272
3273         for (int i = 0; i < vsi->num_queues; i++, que++) {
3274                 struct tx_ring *txr = &que->txr;
3275                 struct rx_ring *rxr = &que->rxr;
3276
3277                 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
3278                         continue;
3279                 IXL_TX_LOCK(txr);
3280                 ixl_free_que_tx(que);
3281                 if (txr->base)
3282                         i40e_free_dma_mem(&pf->hw, &txr->dma);
3283                 IXL_TX_UNLOCK(txr);
3284                 IXL_TX_LOCK_DESTROY(txr);
3285
3286                 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
3287                         continue;
3288                 IXL_RX_LOCK(rxr);
3289                 ixl_free_que_rx(que);
3290                 if (rxr->base)
3291                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
3292                 IXL_RX_UNLOCK(rxr);
3293                 IXL_RX_LOCK_DESTROY(rxr);
3294
3295         }
3296         free(vsi->queues, M_DEVBUF);
3297
3298 free_filters:
3299         /* Free VSI filter list */
3300         ixl_free_mac_filters(vsi);
3301 }
3302
3303 static void
3304 ixl_free_mac_filters(struct ixl_vsi *vsi)
3305 {
3306         struct ixl_mac_filter *f;
3307
3308         while (!SLIST_EMPTY(&vsi->ftl)) {
3309                 f = SLIST_FIRST(&vsi->ftl);
3310                 SLIST_REMOVE_HEAD(&vsi->ftl, next);
3311                 free(f, M_DEVBUF);
3312         }
3313 }
3314
3315
3316 /*********************************************************************
3317  *
3318  *  Allocate memory for the VSI (virtual station interface) and their
3319  *  associated queues, rings and the descriptors associated with each,
3320  *  called only once at attach.
3321  *
3322  **********************************************************************/
3323 static int
3324 ixl_setup_stations(struct ixl_pf *pf)
3325 {
3326         device_t                dev = pf->dev;
3327         struct ixl_vsi          *vsi;
3328         struct ixl_queue        *que;
3329         struct tx_ring          *txr;
3330         struct rx_ring          *rxr;
3331         int                     rsize, tsize;
3332         int                     error = I40E_SUCCESS;
3333
3334         vsi = &pf->vsi;
3335         vsi->back = (void *)pf;
3336         vsi->hw = &pf->hw;
3337         vsi->id = 0;
3338         vsi->num_vlans = 0;
3339         vsi->back = pf;
3340
3341         /* Get memory for the station queues */
3342         if (!(vsi->queues =
3343             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
3344             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3345                 device_printf(dev, "Unable to allocate queue memory\n");
3346                 error = ENOMEM;
3347                 goto early;
3348         }
3349
3350         for (int i = 0; i < vsi->num_queues; i++) {
3351                 que = &vsi->queues[i];
3352                 que->num_desc = ixl_ringsz;
3353                 que->me = i;
3354                 que->vsi = vsi;
3355                 /* mark the queue as active */
3356                 vsi->active_queues |= (u64)1 << que->me;
3357                 txr = &que->txr;
3358                 txr->que = que;
3359                 txr->tail = I40E_QTX_TAIL(que->me);
3360
3361                 /* Initialize the TX lock */
3362                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3363                     device_get_nameunit(dev), que->me);
3364                 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
3365                 /* Create the TX descriptor ring */
3366                 tsize = roundup2((que->num_desc *
3367                     sizeof(struct i40e_tx_desc)) +
3368                     sizeof(u32), DBA_ALIGN);
3369                 if (i40e_allocate_dma_mem(&pf->hw,
3370                     &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
3371                         device_printf(dev,
3372                             "Unable to allocate TX Descriptor memory\n");
3373                         error = ENOMEM;
3374                         goto fail;
3375                 }
3376                 txr->base = (struct i40e_tx_desc *)txr->dma.va;
3377                 bzero((void *)txr->base, tsize);
3378                 /* Now allocate transmit soft structs for the ring */
3379                 if (ixl_allocate_tx_data(que)) {
3380                         device_printf(dev,
3381                             "Critical Failure setting up TX structures\n");
3382                         error = ENOMEM;
3383                         goto fail;
3384                 }
3385                 /* Allocate a buf ring */
3386                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3387                     M_NOWAIT, &txr->mtx);
3388                 if (txr->br == NULL) {
3389                         device_printf(dev,
3390                             "Critical Failure setting up TX buf ring\n");
3391                         error = ENOMEM;
3392                         goto fail;
3393                 }
3394
3395                 /*
3396                  * Next the RX queues...
3397                  */
3398                 rsize = roundup2(que->num_desc *
3399                     sizeof(union i40e_rx_desc), DBA_ALIGN);
3400                 rxr = &que->rxr;
3401                 rxr->que = que;
3402                 rxr->tail = I40E_QRX_TAIL(que->me);
3403
3404                 /* Initialize the RX side lock */
3405                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3406                     device_get_nameunit(dev), que->me);
3407                 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3408
3409                 if (i40e_allocate_dma_mem(&pf->hw,
3410                     &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3411                         device_printf(dev,
3412                             "Unable to allocate RX Descriptor memory\n");
3413                         error = ENOMEM;
3414                         goto fail;
3415                 }
3416                 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3417                 bzero((void *)rxr->base, rsize);
3418
3419                 /* Allocate receive soft structs for the ring*/
3420                 if (ixl_allocate_rx_data(que)) {
3421                         device_printf(dev,
3422                             "Critical Failure setting up receive structs\n");
3423                         error = ENOMEM;
3424                         goto fail;
3425                 }
3426         }
3427
3428         return (0);
3429
3430 fail:
3431         for (int i = 0; i < vsi->num_queues; i++) {
3432                 que = &vsi->queues[i];
3433                 rxr = &que->rxr;
3434                 txr = &que->txr;
3435                 if (rxr->base)
3436                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
3437                 if (txr->base)
3438                         i40e_free_dma_mem(&pf->hw, &txr->dma);
3439         }
3440
3441 early:
3442         return (error);
3443 }
3444
3445 /*
3446 ** Provide a update to the queue RX
3447 ** interrupt moderation value.
3448 */
3449 static void
3450 ixl_set_queue_rx_itr(struct ixl_queue *que)
3451 {
3452         struct ixl_vsi  *vsi = que->vsi;
3453         struct i40e_hw  *hw = vsi->hw;
3454         struct rx_ring  *rxr = &que->rxr;
3455         u16             rx_itr;
3456         u16             rx_latency = 0;
3457         int             rx_bytes;
3458
3459         /* Idle, do nothing */
3460         if (rxr->bytes == 0)
3461                 return;
3462
3463         if (ixl_dynamic_rx_itr) {
3464                 rx_bytes = rxr->bytes/rxr->itr;
3465                 rx_itr = rxr->itr;
3466
3467                 /* Adjust latency range */
3468                 switch (rxr->latency) {
3469                 case IXL_LOW_LATENCY:
3470                         if (rx_bytes > 10) {
3471                                 rx_latency = IXL_AVE_LATENCY;
3472                                 rx_itr = IXL_ITR_20K;
3473                         }
3474                         break;
3475                 case IXL_AVE_LATENCY:
3476                         if (rx_bytes > 20) {
3477                                 rx_latency = IXL_BULK_LATENCY;
3478                                 rx_itr = IXL_ITR_8K;
3479                         } else if (rx_bytes <= 10) {
3480                                 rx_latency = IXL_LOW_LATENCY;
3481                                 rx_itr = IXL_ITR_100K;
3482                         }
3483                         break;
3484                 case IXL_BULK_LATENCY:
3485                         if (rx_bytes <= 20) {
3486                                 rx_latency = IXL_AVE_LATENCY;
3487                                 rx_itr = IXL_ITR_20K;
3488                         }
3489                         break;
3490                  }
3491
3492                 rxr->latency = rx_latency;
3493
3494                 if (rx_itr != rxr->itr) {
3495                         /* do an exponential smoothing */
3496                         rx_itr = (10 * rx_itr * rxr->itr) /
3497                             ((9 * rx_itr) + rxr->itr);
3498                         rxr->itr = rx_itr & IXL_MAX_ITR;
3499                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3500                             que->me), rxr->itr);
3501                 }
3502         } else { /* We may have have toggled to non-dynamic */
3503                 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3504                         vsi->rx_itr_setting = ixl_rx_itr;
3505                 /* Update the hardware if needed */
3506                 if (rxr->itr != vsi->rx_itr_setting) {
3507                         rxr->itr = vsi->rx_itr_setting;
3508                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3509                             que->me), rxr->itr);
3510                 }
3511         }
3512         rxr->bytes = 0;
3513         rxr->packets = 0;
3514         return;
3515 }
3516
3517
3518 /*
3519 ** Provide a update to the queue TX
3520 ** interrupt moderation value.
3521 */
3522 static void
3523 ixl_set_queue_tx_itr(struct ixl_queue *que)
3524 {
3525         struct ixl_vsi  *vsi = que->vsi;
3526         struct i40e_hw  *hw = vsi->hw;
3527         struct tx_ring  *txr = &que->txr;
3528         u16             tx_itr;
3529         u16             tx_latency = 0;
3530         int             tx_bytes;
3531
3532
3533         /* Idle, do nothing */
3534         if (txr->bytes == 0)
3535                 return;
3536
3537         if (ixl_dynamic_tx_itr) {
3538                 tx_bytes = txr->bytes/txr->itr;
3539                 tx_itr = txr->itr;
3540
3541                 switch (txr->latency) {
3542                 case IXL_LOW_LATENCY:
3543                         if (tx_bytes > 10) {
3544                                 tx_latency = IXL_AVE_LATENCY;
3545                                 tx_itr = IXL_ITR_20K;
3546                         }
3547                         break;
3548                 case IXL_AVE_LATENCY:
3549                         if (tx_bytes > 20) {
3550                                 tx_latency = IXL_BULK_LATENCY;
3551                                 tx_itr = IXL_ITR_8K;
3552                         } else if (tx_bytes <= 10) {
3553                                 tx_latency = IXL_LOW_LATENCY;
3554                                 tx_itr = IXL_ITR_100K;
3555                         }
3556                         break;
3557                 case IXL_BULK_LATENCY:
3558                         if (tx_bytes <= 20) {
3559                                 tx_latency = IXL_AVE_LATENCY;
3560                                 tx_itr = IXL_ITR_20K;
3561                         }
3562                         break;
3563                 }
3564
3565                 txr->latency = tx_latency;
3566
3567                 if (tx_itr != txr->itr) {
3568                  /* do an exponential smoothing */
3569                         tx_itr = (10 * tx_itr * txr->itr) /
3570                             ((9 * tx_itr) + txr->itr);
3571                         txr->itr = tx_itr & IXL_MAX_ITR;
3572                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3573                             que->me), txr->itr);
3574                 }
3575
3576         } else { /* We may have have toggled to non-dynamic */
3577                 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3578                         vsi->tx_itr_setting = ixl_tx_itr;
3579                 /* Update the hardware if needed */
3580                 if (txr->itr != vsi->tx_itr_setting) {
3581                         txr->itr = vsi->tx_itr_setting;
3582                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3583                             que->me), txr->itr);
3584                 }
3585         }
3586         txr->bytes = 0;
3587         txr->packets = 0;
3588         return;
3589 }
3590
3591 #define QUEUE_NAME_LEN 32
3592
3593 static void
3594 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3595     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3596 {
3597         struct sysctl_oid *tree;
3598         struct sysctl_oid_list *child;
3599         struct sysctl_oid_list *vsi_list;
3600
3601         tree = device_get_sysctl_tree(pf->dev);
3602         child = SYSCTL_CHILDREN(tree);
3603         vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3604                                    CTLFLAG_RD, NULL, "VSI Number");
3605         vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3606
3607         ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3608 }
3609
3610 #ifdef IXL_DEBUG
3611 /**
3612  * ixl_sysctl_qtx_tail_handler
3613  * Retrieves I40E_QTX_TAIL value from hardware
3614  * for a sysctl.
3615  */
3616 static int
3617 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)
3618 {
3619         struct ixl_queue *que;
3620         int error;
3621         u32 val;
3622
3623         que = ((struct ixl_queue *)oidp->oid_arg1);
3624         if (!que) return 0;
3625
3626         val = rd32(que->vsi->hw, que->txr.tail);
3627         error = sysctl_handle_int(oidp, &val, 0, req);
3628         if (error || !req->newptr)
3629                 return error;
3630         return (0);
3631 }
3632
3633 /**
3634  * ixl_sysctl_qrx_tail_handler
3635  * Retrieves I40E_QRX_TAIL value from hardware
3636  * for a sysctl.
3637  */
3638 static int
3639 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)
3640 {
3641         struct ixl_queue *que;
3642         int error;
3643         u32 val;
3644
3645         que = ((struct ixl_queue *)oidp->oid_arg1);
3646         if (!que) return 0;
3647
3648         val = rd32(que->vsi->hw, que->rxr.tail);
3649         error = sysctl_handle_int(oidp, &val, 0, req);
3650         if (error || !req->newptr)
3651                 return error;
3652         return (0);
3653 }
3654 #endif
3655
3656 static void
3657 ixl_add_hw_stats(struct ixl_pf *pf)
3658 {
3659         device_t dev = pf->dev;
3660         struct ixl_vsi *vsi = &pf->vsi;
3661         struct ixl_queue *queues = vsi->queues;
3662         struct i40e_hw_port_stats *pf_stats = &pf->stats;
3663
3664         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3665         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3666         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3667         struct sysctl_oid_list *vsi_list;
3668
3669         struct sysctl_oid *queue_node;
3670         struct sysctl_oid_list *queue_list;
3671
3672         struct tx_ring *txr;
3673         struct rx_ring *rxr;
3674         char queue_namebuf[QUEUE_NAME_LEN];
3675
3676         /* Driver statistics */
3677         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3678                         CTLFLAG_RD, &pf->watchdog_events,
3679                         "Watchdog timeouts");
3680         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3681                         CTLFLAG_RD, &pf->admin_irq,
3682                         "Admin Queue IRQ Handled");
3683
3684         ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3685         vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3686
3687         /* Queue statistics */
3688         for (int q = 0; q < vsi->num_queues; q++) {
3689                 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3690                 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3691                     OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3692                 queue_list = SYSCTL_CHILDREN(queue_node);
3693
3694                 txr = &(queues[q].txr);
3695                 rxr = &(queues[q].rxr);
3696
3697                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3698                                 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3699                                 "m_defrag() failed");
3700                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3701                                 CTLFLAG_RD, &(queues[q].irqs),
3702                                 "irqs on this queue");
3703                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3704                                 CTLFLAG_RD, &(queues[q].tso),
3705                                 "TSO");
3706                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3707                                 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3708                                 "Driver tx dma failure in xmit");
3709                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3710                                 CTLFLAG_RD, &(txr->no_desc),
3711                                 "Queue No Descriptor Available");
3712                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3713                                 CTLFLAG_RD, &(txr->total_packets),
3714                                 "Queue Packets Transmitted");
3715                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3716                                 CTLFLAG_RD, &(txr->tx_bytes),
3717                                 "Queue Bytes Transmitted");
3718                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3719                                 CTLFLAG_RD, &(rxr->rx_packets),
3720                                 "Queue Packets Received");
3721                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3722                                 CTLFLAG_RD, &(rxr->rx_bytes),
3723                                 "Queue Bytes Received");
3724                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err",
3725                                 CTLFLAG_RD, &(rxr->desc_errs),
3726                                 "Queue Rx Descriptor Errors");
3727                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr",
3728                                 CTLFLAG_RD, &(rxr->itr), 0,
3729                                 "Queue Rx ITR Interval");
3730                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr",
3731                                 CTLFLAG_RD, &(txr->itr), 0,
3732                                 "Queue Tx ITR Interval");
3733                 // Not actual latency; just a calculated value to put in a register
3734                 // TODO: Put in better descriptions here
3735                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_latency",
3736                                 CTLFLAG_RD, &(rxr->latency), 0,
3737                                 "Queue Rx ITRL Average Interval");
3738                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_latency",
3739                                 CTLFLAG_RD, &(txr->latency), 0,
3740                                 "Queue Tx ITRL Average Interval");
3741
3742 #ifdef IXL_DEBUG
3743                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done",
3744                                 CTLFLAG_RD, &(rxr->not_done),
3745                                 "Queue Rx Descriptors not Done");
3746                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh",
3747                                 CTLFLAG_RD, &(rxr->next_refresh), 0,
3748                                 "Queue Rx Descriptors not Done");
3749                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check",
3750                                 CTLFLAG_RD, &(rxr->next_check), 0,
3751                                 "Queue Rx Descriptors not Done");
3752                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail",
3753                                 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3754                                 sizeof(struct ixl_queue),
3755                                 ixl_sysctl_qtx_tail_handler, "IU",
3756                                 "Queue Transmit Descriptor Tail");
3757                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail",
3758                                 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3759                                 sizeof(struct ixl_queue),
3760                                 ixl_sysctl_qrx_tail_handler, "IU",
3761                                 "Queue Receive Descriptor Tail");
3762 #endif
3763         }
3764
3765         /* MAC stats */
3766         ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3767 }
3768
3769 static void
3770 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3771         struct sysctl_oid_list *child,
3772         struct i40e_eth_stats *eth_stats)
3773 {
3774         struct ixl_sysctl_info ctls[] =
3775         {
3776                 {&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3777                 {&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3778                         "Unicast Packets Received"},
3779                 {&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3780                         "Multicast Packets Received"},
3781                 {&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3782                         "Broadcast Packets Received"},
3783                 {&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3784                 {&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3785                 {&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3786                 {&eth_stats->tx_multicast, "mcast_pkts_txd",
3787                         "Multicast Packets Transmitted"},
3788                 {&eth_stats->tx_broadcast, "bcast_pkts_txd",
3789                         "Broadcast Packets Transmitted"},
3790                 // end
3791                 {0,0,0}
3792         };
3793
3794         struct ixl_sysctl_info *entry = ctls;
3795         while (entry->stat != NULL)
3796         {
3797                 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3798                                 CTLFLAG_RD, entry->stat,
3799                                 entry->description);
3800                 entry++;
3801         }
3802 }
3803
3804 static void
3805 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3806         struct sysctl_oid_list *child,
3807         struct i40e_hw_port_stats *stats)
3808 {
3809         struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3810                                     CTLFLAG_RD, NULL, "Mac Statistics");
3811         struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3812
3813         struct i40e_eth_stats *eth_stats = &stats->eth;
3814         ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3815
3816         struct ixl_sysctl_info ctls[] =
3817         {
3818                 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3819                 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3820                 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3821                 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3822                 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3823                 /* Packet Reception Stats */
3824                 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3825                 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3826                 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3827                 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3828                 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3829                 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3830                 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3831                 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3832                 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3833                 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3834                 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3835                 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3836                 /* Packet Transmission Stats */
3837                 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3838                 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3839                 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3840                 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3841                 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3842                 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3843                 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3844                 /* Flow control */
3845                 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3846                 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3847                 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3848                 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3849                 /* End */
3850                 {0,0,0}
3851         };
3852
3853         struct ixl_sysctl_info *entry = ctls;
3854         while (entry->stat != NULL)
3855         {
3856                 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3857                                 CTLFLAG_RD, entry->stat,
3858                                 entry->description);
3859                 entry++;
3860         }
3861 }
3862
3863
3864 /*
3865 ** ixl_config_rss - setup RSS
3866 **  - note this is done for the single vsi
3867 */
3868 static void
3869 ixl_config_rss(struct ixl_vsi *vsi)
3870 {
3871         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3872         struct i40e_hw  *hw = vsi->hw;
3873         u32             lut = 0;
3874         u64             set_hena = 0, hena;
3875         int             i, j, que_id;
3876 #ifdef RSS
3877         u32             rss_hash_config;
3878         u32             rss_seed[IXL_KEYSZ];
3879 #else
3880         u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3881                             0x183cfd8c, 0xce880440, 0x580cbc3c,
3882                             0x35897377, 0x328b25e1, 0x4fa98922,
3883                             0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3884 #endif
3885
3886 #ifdef RSS
3887         /* Fetch the configured RSS key */
3888         rss_getkey((uint8_t *) &rss_seed);
3889 #endif
3890
3891         /* Fill out hash function seed */
3892         for (i = 0; i < IXL_KEYSZ; i++)
3893                 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3894
3895         /* Enable PCTYPES for RSS: */
3896 #ifdef RSS
3897         rss_hash_config = rss_gethashconfig();
3898         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3899                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3900         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3901                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3902         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3903                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3904         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3905                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3906         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3907                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3908         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3909                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3910         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3911                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3912 #else
3913         set_hena =
3914                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3915                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3916                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3917                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3918                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3919                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3920                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3921                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3922                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3923                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3924                 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3925 #endif
3926         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
3927             ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
3928         hena |= set_hena;
3929         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
3930         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3931
3932         /* Populate the LUT with max no. of queues in round robin fashion */
3933         for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3934                 if (j == vsi->num_queues)
3935                         j = 0;
3936 #ifdef RSS
3937                 /*
3938                  * Fetch the RSS bucket id for the given indirection entry.
3939                  * Cap it at the number of configured buckets (which is
3940                  * num_queues.)
3941                  */
3942                 que_id = rss_get_indirection_to_bucket(i);
3943                 que_id = que_id % vsi->num_queues;
3944 #else
3945                 que_id = j;
3946 #endif
3947                 /* lut = 4-byte sliding window of 4 lut entries */
3948                 lut = (lut << 8) | (que_id &
3949                     ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3950                 /* On i = 3, we have 4 entries in lut; write to the register */
3951                 if ((i & 3) == 3)
3952                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3953         }
3954         ixl_flush(hw);
3955 }
3956
3957
3958 /*
3959 ** This routine is run via an vlan config EVENT,
3960 ** it enables us to use the HW Filter table since
3961 ** we can get the vlan id. This just creates the
3962 ** entry in the soft version of the VFTA, init will
3963 ** repopulate the real table.
3964 */
3965 static void
3966 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3967 {
3968         struct ixl_vsi  *vsi = ifp->if_softc;
3969         struct i40e_hw  *hw = vsi->hw;
3970         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3971
3972         if (ifp->if_softc !=  arg)   /* Not our event */
3973                 return;
3974
3975         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3976                 return;
3977
3978         IXL_PF_LOCK(pf);
3979         ++vsi->num_vlans;
3980         ixl_add_filter(vsi, hw->mac.addr, vtag);
3981         IXL_PF_UNLOCK(pf);
3982 }
3983
3984 /*
3985 ** This routine is run via an vlan
3986 ** unconfig EVENT, remove our entry
3987 ** in the soft vfta.
3988 */
3989 static void
3990 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3991 {
3992         struct ixl_vsi  *vsi = ifp->if_softc;
3993         struct i40e_hw  *hw = vsi->hw;
3994         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3995
3996         if (ifp->if_softc !=  arg)
3997                 return;
3998
3999         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4000                 return;
4001
4002         IXL_PF_LOCK(pf);
4003         --vsi->num_vlans;
4004         ixl_del_filter(vsi, hw->mac.addr, vtag);
4005         IXL_PF_UNLOCK(pf);
4006 }
4007
4008 /*
4009 ** This routine updates vlan filters, called by init
4010 ** it scans the filter table and then updates the hw
4011 ** after a soft reset.
4012 */
4013 static void
4014 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
4015 {
4016         struct ixl_mac_filter   *f;
4017         int                     cnt = 0, flags;
4018
4019         if (vsi->num_vlans == 0)
4020                 return;
4021         /*
4022         ** Scan the filter list for vlan entries,
4023         ** mark them for addition and then call
4024         ** for the AQ update.
4025         */
4026         SLIST_FOREACH(f, &vsi->ftl, next) {
4027                 if (f->flags & IXL_FILTER_VLAN) {
4028                         f->flags |=
4029                             (IXL_FILTER_ADD |
4030                             IXL_FILTER_USED);
4031                         cnt++;
4032                 }
4033         }
4034         if (cnt == 0) {
4035                 printf("setup vlan: no filters found!\n");
4036                 return;
4037         }
4038         flags = IXL_FILTER_VLAN;
4039         flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4040         ixl_add_hw_filters(vsi, flags, cnt);
4041         return;
4042 }
4043
4044 /*
4045 ** Initialize filter list and add filters that the hardware
4046 ** needs to know about.
4047 **
4048 ** Requires VSI's filter list & seid to be set before calling.
4049 */
4050 static void
4051 ixl_init_filters(struct ixl_vsi *vsi)
4052 {
4053         /* Add broadcast address */
4054         ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
4055
4056         /*
4057          * Prevent Tx flow control frames from being sent out by
4058          * non-firmware transmitters.
4059          */
4060         i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid);
4061 }
4062
4063 /*
4064 ** This routine adds mulicast filters
4065 */
4066 static void
4067 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
4068 {
4069         struct ixl_mac_filter *f;
4070
4071         /* Does one already exist */
4072         f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4073         if (f != NULL)
4074                 return;
4075
4076         f = ixl_get_filter(vsi);
4077         if (f == NULL) {
4078                 printf("WARNING: no filter available!!\n");
4079                 return;
4080         }
4081         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4082         f->vlan = IXL_VLAN_ANY;
4083         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
4084             | IXL_FILTER_MC);
4085
4086         return;
4087 }
4088
4089 static void
4090 ixl_reconfigure_filters(struct ixl_vsi *vsi)
4091 {
4092
4093         ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
4094 }
4095
4096 /*
4097 ** This routine adds macvlan filters
4098 */
4099 static void
4100 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4101 {
4102         struct ixl_mac_filter   *f, *tmp;
4103         struct ixl_pf           *pf;
4104         device_t                dev;
4105
4106         DEBUGOUT("ixl_add_filter: begin");
4107
4108         pf = vsi->back;
4109         dev = pf->dev;
4110
4111         /* Does one already exist */
4112         f = ixl_find_filter(vsi, macaddr, vlan);
4113         if (f != NULL)
4114                 return;
4115         /*
4116         ** Is this the first vlan being registered, if so we
4117         ** need to remove the ANY filter that indicates we are
4118         ** not in a vlan, and replace that with a 0 filter.
4119         */
4120         if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
4121                 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4122                 if (tmp != NULL) {
4123                         ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
4124                         ixl_add_filter(vsi, macaddr, 0);
4125                 }
4126         }
4127
4128         f = ixl_get_filter(vsi);
4129         if (f == NULL) {
4130                 device_printf(dev, "WARNING: no filter available!!\n");
4131                 return;
4132         }
4133         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4134         f->vlan = vlan;
4135         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4136         if (f->vlan != IXL_VLAN_ANY)
4137                 f->flags |= IXL_FILTER_VLAN;
4138         else
4139                 vsi->num_macs++;
4140
4141         ixl_add_hw_filters(vsi, f->flags, 1);
4142         return;
4143 }
4144
4145 static void
4146 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4147 {
4148         struct ixl_mac_filter *f;
4149
4150         f = ixl_find_filter(vsi, macaddr, vlan);
4151         if (f == NULL)
4152                 return;
4153
4154         f->flags |= IXL_FILTER_DEL;
4155         ixl_del_hw_filters(vsi, 1);
4156         vsi->num_macs--;
4157
4158         /* Check if this is the last vlan removal */
4159         if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
4160                 /* Switch back to a non-vlan filter */
4161                 ixl_del_filter(vsi, macaddr, 0);
4162                 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
4163         }
4164         return;
4165 }
4166
4167 /*
4168 ** Find the filter with both matching mac addr and vlan id
4169 */
4170 static struct ixl_mac_filter *
4171 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4172 {
4173         struct ixl_mac_filter   *f;
4174         bool                    match = FALSE;
4175
4176         SLIST_FOREACH(f, &vsi->ftl, next) {
4177                 if (!cmp_etheraddr(f->macaddr, macaddr))
4178                         continue;
4179                 if (f->vlan == vlan) {
4180                         match = TRUE;
4181                         break;
4182                 }
4183         }
4184
4185         if (!match)
4186                 f = NULL;
4187         return (f);
4188 }
4189
4190 /*
4191 ** This routine takes additions to the vsi filter
4192 ** table and creates an Admin Queue call to create
4193 ** the filters in the hardware.
4194 */
4195 static void
4196 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
4197 {
4198         struct i40e_aqc_add_macvlan_element_data *a, *b;
4199         struct ixl_mac_filter   *f;
4200         struct ixl_pf           *pf;
4201         struct i40e_hw          *hw;
4202         device_t                dev;
4203         int                     err, j = 0;
4204
4205         pf = vsi->back;
4206         dev = pf->dev;
4207         hw = &pf->hw;
4208         IXL_PF_LOCK_ASSERT(pf);
4209
4210         a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
4211             M_DEVBUF, M_NOWAIT | M_ZERO);
4212         if (a == NULL) {
4213                 device_printf(dev, "add_hw_filters failed to get memory\n");
4214                 return;
4215         }
4216
4217         /*
4218         ** Scan the filter list, each time we find one
4219         ** we add it to the admin queue array and turn off
4220         ** the add bit.
4221         */
4222         SLIST_FOREACH(f, &vsi->ftl, next) {
4223                 if (f->flags == flags) {
4224                         b = &a[j]; // a pox on fvl long names :)
4225                         bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
4226                         if (f->vlan == IXL_VLAN_ANY) {
4227                                 b->vlan_tag = 0;
4228                                 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
4229                         } else {
4230                                 b->vlan_tag = f->vlan;
4231                                 b->flags = 0;
4232                         }
4233                         b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
4234                         f->flags &= ~IXL_FILTER_ADD;
4235                         j++;
4236                 }
4237                 if (j == cnt)
4238                         break;
4239         }
4240         if (j > 0) {
4241                 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
4242                 if (err)
4243                         device_printf(dev, "aq_add_macvlan err %d, "
4244                             "aq_error %d\n", err, hw->aq.asq_last_status);
4245                 else
4246                         vsi->hw_filters_add += j;
4247         }
4248         free(a, M_DEVBUF);
4249         return;
4250 }
4251
4252 /*
4253 ** This routine takes removals in the vsi filter
4254 ** table and creates an Admin Queue call to delete
4255 ** the filters in the hardware.
4256 */
4257 static void
4258 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
4259 {
4260         struct i40e_aqc_remove_macvlan_element_data *d, *e;
4261         struct ixl_pf           *pf;
4262         struct i40e_hw          *hw;
4263         device_t                dev;
4264         struct ixl_mac_filter   *f, *f_temp;
4265         int                     err, j = 0;
4266
4267         DEBUGOUT("ixl_del_hw_filters: begin\n");
4268
4269         pf = vsi->back;
4270         hw = &pf->hw;
4271         dev = pf->dev;
4272
4273         d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
4274             M_DEVBUF, M_NOWAIT | M_ZERO);
4275         if (d == NULL) {
4276                 printf("del hw filter failed to get memory\n");
4277                 return;
4278         }
4279
4280         SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
4281                 if (f->flags & IXL_FILTER_DEL) {
4282                         e = &d[j]; // a pox on fvl long names :)
4283                         bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
4284                         e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
4285                         e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
4286                         /* delete entry from vsi list */
4287                         SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
4288                         free(f, M_DEVBUF);
4289                         j++;
4290                 }
4291                 if (j == cnt)
4292                         break;
4293         }
4294         if (j > 0) {
4295                 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
4296                 /* NOTE: returns ENOENT every time but seems to work fine,
4297                    so we'll ignore that specific error. */
4298                 // TODO: Does this still occur on current firmwares?
4299                 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
4300                         int sc = 0;
4301                         for (int i = 0; i < j; i++)
4302                                 sc += (!d[i].error_code);
4303                         vsi->hw_filters_del += sc;
4304                         device_printf(dev,
4305                             "Failed to remove %d/%d filters, aq error %d\n",
4306                             j - sc, j, hw->aq.asq_last_status);
4307                 } else
4308                         vsi->hw_filters_del += j;
4309         }
4310         free(d, M_DEVBUF);
4311
4312         DEBUGOUT("ixl_del_hw_filters: end\n");
4313         return;
4314 }
4315
4316 static int
4317 ixl_enable_rings(struct ixl_vsi *vsi)
4318 {
4319         struct ixl_pf   *pf = vsi->back;
4320         struct i40e_hw  *hw = &pf->hw;
4321         int             index, error;
4322         u32             reg;
4323
4324         error = 0;
4325         for (int i = 0; i < vsi->num_queues; i++) {
4326                 index = vsi->first_queue + i;
4327                 i40e_pre_tx_queue_cfg(hw, index, TRUE);
4328
4329                 reg = rd32(hw, I40E_QTX_ENA(index));
4330                 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
4331                     I40E_QTX_ENA_QENA_STAT_MASK;
4332                 wr32(hw, I40E_QTX_ENA(index), reg);
4333                 /* Verify the enable took */
4334                 for (int j = 0; j < 10; j++) {
4335                         reg = rd32(hw, I40E_QTX_ENA(index));
4336                         if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4337                                 break;
4338                         i40e_msec_delay(10);
4339                 }
4340                 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
4341                         device_printf(pf->dev, "TX queue %d disabled!\n",
4342                             index);
4343                         error = ETIMEDOUT;
4344                 }
4345
4346                 reg = rd32(hw, I40E_QRX_ENA(index));
4347                 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
4348                     I40E_QRX_ENA_QENA_STAT_MASK;
4349                 wr32(hw, I40E_QRX_ENA(index), reg);
4350                 /* Verify the enable took */
4351                 for (int j = 0; j < 10; j++) {
4352                         reg = rd32(hw, I40E_QRX_ENA(index));
4353                         if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
4354                                 break;
4355                         i40e_msec_delay(10);
4356                 }
4357                 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
4358                         device_printf(pf->dev, "RX queue %d disabled!\n",
4359                             index);
4360                         error = ETIMEDOUT;
4361                 }
4362         }
4363
4364         return (error);
4365 }
4366
4367 static int
4368 ixl_disable_rings(struct ixl_vsi *vsi)
4369 {
4370         struct ixl_pf   *pf = vsi->back;
4371         struct i40e_hw  *hw = &pf->hw;
4372         int             index, error;
4373         u32             reg;
4374
4375         error = 0;
4376         for (int i = 0; i < vsi->num_queues; i++) {
4377                 index = vsi->first_queue + i;
4378
4379                 i40e_pre_tx_queue_cfg(hw, index, FALSE);
4380                 i40e_usec_delay(500);
4381
4382                 reg = rd32(hw, I40E_QTX_ENA(index));
4383                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4384                 wr32(hw, I40E_QTX_ENA(index), reg);
4385                 /* Verify the disable took */
4386                 for (int j = 0; j < 10; j++) {
4387                         reg = rd32(hw, I40E_QTX_ENA(index));
4388                         if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4389                                 break;
4390                         i40e_msec_delay(10);
4391                 }
4392                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
4393                         device_printf(pf->dev, "TX queue %d still enabled!\n",
4394                             index);
4395                         error = ETIMEDOUT;
4396                 }
4397
4398                 reg = rd32(hw, I40E_QRX_ENA(index));
4399                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4400                 wr32(hw, I40E_QRX_ENA(index), reg);
4401                 /* Verify the disable took */
4402                 for (int j = 0; j < 10; j++) {
4403                         reg = rd32(hw, I40E_QRX_ENA(index));
4404                         if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
4405                                 break;
4406                         i40e_msec_delay(10);
4407                 }
4408                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
4409                         device_printf(pf->dev, "RX queue %d still enabled!\n",
4410                             index);
4411                         error = ETIMEDOUT;
4412                 }
4413         }
4414
4415         return (error);
4416 }
4417
4418 /**
4419  * ixl_handle_mdd_event
4420  *
4421  * Called from interrupt handler to identify possibly malicious vfs
4422  * (But also detects events from the PF, as well)
4423  **/
4424 static void
4425 ixl_handle_mdd_event(struct ixl_pf *pf)
4426 {
4427         struct i40e_hw *hw = &pf->hw;
4428         device_t dev = pf->dev;
4429         bool mdd_detected = false;
4430         bool pf_mdd_detected = false;
4431         u32 reg;
4432
4433         /* find what triggered the MDD event */
4434         reg = rd32(hw, I40E_GL_MDET_TX);
4435         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
4436                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
4437                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
4438                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
4439                                 I40E_GL_MDET_TX_EVENT_SHIFT;
4440                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
4441                                 I40E_GL_MDET_TX_QUEUE_SHIFT;
4442                 device_printf(dev,
4443                          "Malicious Driver Detection event 0x%02x"
4444                          " on TX queue %d pf number 0x%02x\n",
4445                          event, queue, pf_num);
4446                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
4447                 mdd_detected = true;
4448         }
4449         reg = rd32(hw, I40E_GL_MDET_RX);
4450         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4451                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
4452                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
4453                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
4454                                 I40E_GL_MDET_RX_EVENT_SHIFT;
4455                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
4456                                 I40E_GL_MDET_RX_QUEUE_SHIFT;
4457                 device_printf(dev,
4458                          "Malicious Driver Detection event 0x%02x"
4459                          " on RX queue %d of function 0x%02x\n",
4460                          event, queue, func);
4461                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4462                 mdd_detected = true;
4463         }
4464
4465         if (mdd_detected) {
4466                 reg = rd32(hw, I40E_PF_MDET_TX);
4467                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4468                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4469                         device_printf(dev,
4470                                  "MDD TX event is for this function 0x%08x",
4471                                  reg);
4472                         pf_mdd_detected = true;
4473                 }
4474                 reg = rd32(hw, I40E_PF_MDET_RX);
4475                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4476                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4477                         device_printf(dev,
4478                                  "MDD RX event is for this function 0x%08x",
4479                                  reg);
4480                         pf_mdd_detected = true;
4481                 }
4482         }
4483
4484         /* re-enable mdd interrupt cause */
4485         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4486         reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4487         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4488         ixl_flush(hw);
4489 }
4490
4491 static void
4492 ixl_enable_intr(struct ixl_vsi *vsi)
4493 {
4494         struct i40e_hw          *hw = vsi->hw;
4495         struct ixl_queue        *que = vsi->queues;
4496
4497         if (ixl_enable_msix) {
4498                 for (int i = 0; i < vsi->num_queues; i++, que++)
4499                         ixl_enable_queue(hw, que->me);
4500         } else
4501                 ixl_enable_legacy(hw);
4502 }
4503
4504 static void
4505 ixl_disable_rings_intr(struct ixl_vsi *vsi)
4506 {
4507         struct i40e_hw          *hw = vsi->hw;
4508         struct ixl_queue        *que = vsi->queues;
4509
4510         for (int i = 0; i < vsi->num_queues; i++, que++)
4511                 ixl_disable_queue(hw, que->me);
4512 }
4513
4514 static void
4515 ixl_disable_intr(struct ixl_vsi *vsi)
4516 {
4517         struct i40e_hw          *hw = vsi->hw;
4518
4519         if (ixl_enable_msix)
4520                 ixl_disable_adminq(hw);
4521         else
4522                 ixl_disable_legacy(hw);
4523 }
4524
4525 static void
4526 ixl_enable_adminq(struct i40e_hw *hw)
4527 {
4528         u32             reg;
4529
4530         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4531             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4532             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4533         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4534         ixl_flush(hw);
4535 }
4536
4537 static void
4538 ixl_disable_adminq(struct i40e_hw *hw)
4539 {
4540         u32             reg;
4541
4542         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4543         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4544         ixl_flush(hw);
4545 }
4546
4547 static void
4548 ixl_enable_queue(struct i40e_hw *hw, int id)
4549 {
4550         u32             reg;
4551
4552         reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4553             I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4554             (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4555         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4556 }
4557
4558 static void
4559 ixl_disable_queue(struct i40e_hw *hw, int id)
4560 {
4561         u32             reg;
4562
4563         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4564         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4565 }
4566
4567 static void
4568 ixl_enable_legacy(struct i40e_hw *hw)
4569 {
4570         u32             reg;
4571         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4572             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4573             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4574         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4575 }
4576
4577 static void
4578 ixl_disable_legacy(struct i40e_hw *hw)
4579 {
4580         u32             reg;
4581
4582         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4583         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4584 }
4585
4586 static void
4587 ixl_update_stats_counters(struct ixl_pf *pf)
4588 {
4589         struct i40e_hw  *hw = &pf->hw;
4590         struct ixl_vsi  *vsi = &pf->vsi;
4591         struct ixl_vf   *vf;
4592
4593         struct i40e_hw_port_stats *nsd = &pf->stats;
4594         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4595
4596         /* Update hw stats */
4597         ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4598                            pf->stat_offsets_loaded,
4599                            &osd->crc_errors, &nsd->crc_errors);
4600         ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4601                            pf->stat_offsets_loaded,
4602                            &osd->illegal_bytes, &nsd->illegal_bytes);
4603         ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4604                            I40E_GLPRT_GORCL(hw->port),
4605                            pf->stat_offsets_loaded,
4606                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4607         ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4608                            I40E_GLPRT_GOTCL(hw->port),
4609                            pf->stat_offsets_loaded,
4610                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4611         ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4612                            pf->stat_offsets_loaded,
4613                            &osd->eth.rx_discards,
4614                            &nsd->eth.rx_discards);
4615         ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4616                            I40E_GLPRT_UPRCL(hw->port),
4617                            pf->stat_offsets_loaded,
4618                            &osd->eth.rx_unicast,
4619                            &nsd->eth.rx_unicast);
4620         ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4621                            I40E_GLPRT_UPTCL(hw->port),
4622                            pf->stat_offsets_loaded,
4623                            &osd->eth.tx_unicast,
4624                            &nsd->eth.tx_unicast);
4625         ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4626                            I40E_GLPRT_MPRCL(hw->port),
4627                            pf->stat_offsets_loaded,
4628                            &osd->eth.rx_multicast,
4629                            &nsd->eth.rx_multicast);
4630         ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4631                            I40E_GLPRT_MPTCL(hw->port),
4632                            pf->stat_offsets_loaded,
4633                            &osd->eth.tx_multicast,
4634                            &nsd->eth.tx_multicast);
4635         ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4636                            I40E_GLPRT_BPRCL(hw->port),
4637                            pf->stat_offsets_loaded,
4638                            &osd->eth.rx_broadcast,
4639                            &nsd->eth.rx_broadcast);
4640         ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4641                            I40E_GLPRT_BPTCL(hw->port),
4642                            pf->stat_offsets_loaded,
4643                            &osd->eth.tx_broadcast,
4644                            &nsd->eth.tx_broadcast);
4645
4646         ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4647                            pf->stat_offsets_loaded,
4648                            &osd->tx_dropped_link_down,
4649                            &nsd->tx_dropped_link_down);
4650         ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4651                            pf->stat_offsets_loaded,
4652                            &osd->mac_local_faults,
4653                            &nsd->mac_local_faults);
4654         ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4655                            pf->stat_offsets_loaded,
4656                            &osd->mac_remote_faults,
4657                            &nsd->mac_remote_faults);
4658         ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4659                            pf->stat_offsets_loaded,
4660                            &osd->rx_length_errors,
4661                            &nsd->rx_length_errors);
4662
4663         /* Flow control (LFC) stats */
4664         ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4665                            pf->stat_offsets_loaded,
4666                            &osd->link_xon_rx, &nsd->link_xon_rx);
4667         ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4668                            pf->stat_offsets_loaded,
4669                            &osd->link_xon_tx, &nsd->link_xon_tx);
4670         ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4671                            pf->stat_offsets_loaded,
4672                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
4673         ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4674                            pf->stat_offsets_loaded,
4675                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
4676
4677         /* Packet size stats rx */
4678         ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4679                            I40E_GLPRT_PRC64L(hw->port),
4680                            pf->stat_offsets_loaded,
4681                            &osd->rx_size_64, &nsd->rx_size_64);
4682         ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4683                            I40E_GLPRT_PRC127L(hw->port),
4684                            pf->stat_offsets_loaded,
4685                            &osd->rx_size_127, &nsd->rx_size_127);
4686         ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4687                            I40E_GLPRT_PRC255L(hw->port),
4688                            pf->stat_offsets_loaded,
4689                            &osd->rx_size_255, &nsd->rx_size_255);
4690         ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4691                            I40E_GLPRT_PRC511L(hw->port),
4692                            pf->stat_offsets_loaded,
4693                            &osd->rx_size_511, &nsd->rx_size_511);
4694         ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4695                            I40E_GLPRT_PRC1023L(hw->port),
4696                            pf->stat_offsets_loaded,
4697                            &osd->rx_size_1023, &nsd->rx_size_1023);
4698         ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4699                            I40E_GLPRT_PRC1522L(hw->port),
4700                            pf->stat_offsets_loaded,
4701                            &osd->rx_size_1522, &nsd->rx_size_1522);
4702         ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4703                            I40E_GLPRT_PRC9522L(hw->port),
4704                            pf->stat_offsets_loaded,
4705                            &osd->rx_size_big, &nsd->rx_size_big);
4706
4707         /* Packet size stats tx */
4708         ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4709                            I40E_GLPRT_PTC64L(hw->port),
4710                            pf->stat_offsets_loaded,
4711                            &osd->tx_size_64, &nsd->tx_size_64);
4712         ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4713                            I40E_GLPRT_PTC127L(hw->port),
4714                            pf->stat_offsets_loaded,
4715                            &osd->tx_size_127, &nsd->tx_size_127);
4716         ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4717                            I40E_GLPRT_PTC255L(hw->port),
4718                            pf->stat_offsets_loaded,
4719                            &osd->tx_size_255, &nsd->tx_size_255);
4720         ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4721                            I40E_GLPRT_PTC511L(hw->port),
4722                            pf->stat_offsets_loaded,
4723                            &osd->tx_size_511, &nsd->tx_size_511);
4724         ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4725                            I40E_GLPRT_PTC1023L(hw->port),
4726                            pf->stat_offsets_loaded,
4727                            &osd->tx_size_1023, &nsd->tx_size_1023);
4728         ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4729                            I40E_GLPRT_PTC1522L(hw->port),
4730                            pf->stat_offsets_loaded,
4731                            &osd->tx_size_1522, &nsd->tx_size_1522);
4732         ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4733                            I40E_GLPRT_PTC9522L(hw->port),
4734                            pf->stat_offsets_loaded,
4735                            &osd->tx_size_big, &nsd->tx_size_big);
4736
4737         ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4738                            pf->stat_offsets_loaded,
4739                            &osd->rx_undersize, &nsd->rx_undersize);
4740         ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4741                            pf->stat_offsets_loaded,
4742                            &osd->rx_fragments, &nsd->rx_fragments);
4743         ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4744                            pf->stat_offsets_loaded,
4745                            &osd->rx_oversize, &nsd->rx_oversize);
4746         ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4747                            pf->stat_offsets_loaded,
4748                            &osd->rx_jabber, &nsd->rx_jabber);
4749         pf->stat_offsets_loaded = true;
4750         /* End hw stats */
4751
4752         /* Update vsi stats */
4753         ixl_update_vsi_stats(vsi);
4754
4755         for (int i = 0; i < pf->num_vfs; i++) {
4756                 vf = &pf->vfs[i];
4757                 if (vf->vf_flags & VF_FLAG_ENABLED)
4758                         ixl_update_eth_stats(&pf->vfs[i].vsi);
4759         }
4760 }
4761
4762 static int
4763 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf)
4764 {
4765         struct i40e_hw *hw = &pf->hw;
4766         struct ixl_vsi *vsi = &pf->vsi;
4767         device_t dev = pf->dev;
4768         bool is_up = false;
4769         int error = 0;
4770
4771         is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING);
4772
4773         /* Teardown */
4774         if (is_up)
4775                 ixl_stop(pf);
4776         error = i40e_shutdown_lan_hmc(hw);
4777         if (error)
4778                 device_printf(dev,
4779                     "Shutdown LAN HMC failed with code %d\n", error);
4780         ixl_disable_adminq(hw);
4781         ixl_teardown_adminq_msix(pf);
4782         error = i40e_shutdown_adminq(hw);
4783         if (error)
4784                 device_printf(dev,
4785                     "Shutdown Admin queue failed with code %d\n", error);
4786
4787         /* Setup */
4788         error = i40e_init_adminq(hw);
4789         if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
4790                 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
4791                     error);
4792         }
4793         error = ixl_setup_adminq_msix(pf);
4794         if (error) {
4795                 device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
4796                     error);
4797         }
4798         ixl_configure_intr0_msix(pf);
4799         ixl_enable_adminq(hw);
4800         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4801             hw->func_caps.num_rx_qp, 0, 0);
4802         if (error) {
4803                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
4804         }
4805         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4806         if (error) {
4807                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
4808         }
4809         if (is_up)
4810                 ixl_init(pf);
4811
4812         return (0);
4813 }
4814
4815 static void
4816 ixl_handle_empr_reset(struct ixl_pf *pf)
4817 {
4818         struct i40e_hw *hw = &pf->hw;
4819         device_t dev = pf->dev;
4820         int count = 0;
4821         u32 reg;
4822
4823         /* Typically finishes within 3-4 seconds */
4824         while (count++ < 100) {
4825                 reg = rd32(hw, I40E_GLGEN_RSTAT)
4826                     & I40E_GLGEN_RSTAT_DEVSTATE_MASK;
4827                 if (reg)
4828                         i40e_msec_delay(100);
4829                 else
4830                         break;
4831         }
4832 #ifdef IXL_DEBUG
4833         // Reset-related
4834         device_printf(dev, "EMPR reset wait count: %d\n", count);
4835 #endif
4836
4837         device_printf(dev, "Rebuilding driver state...\n");
4838         ixl_rebuild_hw_structs_after_reset(pf);
4839         device_printf(dev, "Rebuilding driver state done.\n");
4840
4841         atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
4842 }
4843
4844 /*
4845 ** Tasklet handler for MSIX Adminq interrupts
4846 **  - do outside interrupt since it might sleep
4847 */
4848 static void
4849 ixl_do_adminq(void *context, int pending)
4850 {
4851         struct ixl_pf                   *pf = context;
4852         struct i40e_hw                  *hw = &pf->hw;
4853         struct i40e_arq_event_info      event;
4854         i40e_status                     ret;
4855         device_t                        dev = pf->dev;
4856         u32                             loop = 0;
4857         u16                             opcode, result;
4858
4859         if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
4860                 /* Flag cleared at end of this function */
4861                 ixl_handle_empr_reset(pf);
4862                 return;
4863         }
4864
4865         /* Admin Queue handling */
4866         event.buf_len = IXL_AQ_BUF_SZ;
4867         event.msg_buf = malloc(event.buf_len,
4868             M_DEVBUF, M_NOWAIT | M_ZERO);
4869         if (!event.msg_buf) {
4870                 device_printf(dev, "%s: Unable to allocate memory for Admin"
4871                     " Queue event!\n", __func__);
4872                 return;
4873         }
4874
4875         IXL_PF_LOCK(pf);
4876         /* clean and process any events */
4877         do {
4878                 ret = i40e_clean_arq_element(hw, &event, &result);
4879                 if (ret)
4880                         break;
4881                 opcode = LE16_TO_CPU(event.desc.opcode);
4882 #ifdef IXL_DEBUG
4883                 device_printf(dev, "%s: Admin Queue event: %#06x\n", __func__,
4884                     opcode);
4885 #endif
4886                 switch (opcode) {
4887                 case i40e_aqc_opc_get_link_status:
4888                         ixl_link_event(pf, &event);
4889                         break;
4890                 case i40e_aqc_opc_send_msg_to_pf:
4891 #ifdef PCI_IOV
4892                         ixl_handle_vf_msg(pf, &event);
4893 #endif
4894                         break;
4895                 case i40e_aqc_opc_event_lan_overflow:
4896                 default:
4897                         break;
4898                 }
4899
4900         } while (result && (loop++ < IXL_ADM_LIMIT));
4901
4902         free(event.msg_buf, M_DEVBUF);
4903
4904         /*
4905          * If there are still messages to process, reschedule ourselves.
4906          * Otherwise, re-enable our interrupt and go to sleep.
4907          */
4908         if (result > 0)
4909                 taskqueue_enqueue(pf->tq, &pf->adminq);
4910         else
4911                 ixl_enable_adminq(hw);
4912
4913         IXL_PF_UNLOCK(pf);
4914 }
4915
4916 /**
4917  * Update VSI-specific ethernet statistics counters.
4918  **/
4919 void
4920 ixl_update_eth_stats(struct ixl_vsi *vsi)
4921 {
4922         struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4923         struct i40e_hw *hw = &pf->hw;
4924         struct i40e_eth_stats *es;
4925         struct i40e_eth_stats *oes;
4926         struct i40e_hw_port_stats *nsd;
4927         u16 stat_idx = vsi->info.stat_counter_idx;
4928
4929         es = &vsi->eth_stats;
4930         oes = &vsi->eth_stats_offsets;
4931         nsd = &pf->stats;
4932
4933         /* Gather up the stats that the hw collects */
4934         ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4935                            vsi->stat_offsets_loaded,
4936                            &oes->tx_errors, &es->tx_errors);
4937         ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4938                            vsi->stat_offsets_loaded,
4939                            &oes->rx_discards, &es->rx_discards);
4940
4941         ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4942                            I40E_GLV_GORCL(stat_idx),
4943                            vsi->stat_offsets_loaded,
4944                            &oes->rx_bytes, &es->rx_bytes);
4945         ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4946                            I40E_GLV_UPRCL(stat_idx),
4947                            vsi->stat_offsets_loaded,
4948                            &oes->rx_unicast, &es->rx_unicast);
4949         ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4950                            I40E_GLV_MPRCL(stat_idx),
4951                            vsi->stat_offsets_loaded,
4952                            &oes->rx_multicast, &es->rx_multicast);
4953         ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4954                            I40E_GLV_BPRCL(stat_idx),
4955                            vsi->stat_offsets_loaded,
4956                            &oes->rx_broadcast, &es->rx_broadcast);
4957
4958         ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4959                            I40E_GLV_GOTCL(stat_idx),
4960                            vsi->stat_offsets_loaded,
4961                            &oes->tx_bytes, &es->tx_bytes);
4962         ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4963                            I40E_GLV_UPTCL(stat_idx),
4964                            vsi->stat_offsets_loaded,
4965                            &oes->tx_unicast, &es->tx_unicast);
4966         ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4967                            I40E_GLV_MPTCL(stat_idx),
4968                            vsi->stat_offsets_loaded,
4969                            &oes->tx_multicast, &es->tx_multicast);
4970         ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4971                            I40E_GLV_BPTCL(stat_idx),
4972                            vsi->stat_offsets_loaded,
4973                            &oes->tx_broadcast, &es->tx_broadcast);
4974         vsi->stat_offsets_loaded = true;
4975 }
4976
4977 static void
4978 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4979 {
4980         struct ixl_pf           *pf;
4981         struct ifnet            *ifp;
4982         struct i40e_eth_stats   *es;
4983         u64                     tx_discards;
4984
4985         struct i40e_hw_port_stats *nsd;
4986
4987         pf = vsi->back;
4988         ifp = vsi->ifp;
4989         es = &vsi->eth_stats;
4990         nsd = &pf->stats;
4991
4992         ixl_update_eth_stats(vsi);
4993
4994         tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4995         for (int i = 0; i < vsi->num_queues; i++)
4996                 tx_discards += vsi->queues[i].txr.br->br_drops;
4997
4998         /* Update ifnet stats */
4999         IXL_SET_IPACKETS(vsi, es->rx_unicast +
5000                            es->rx_multicast +
5001                            es->rx_broadcast);
5002         IXL_SET_OPACKETS(vsi, es->tx_unicast +
5003                            es->tx_multicast +
5004                            es->tx_broadcast);
5005         IXL_SET_IBYTES(vsi, es->rx_bytes);
5006         IXL_SET_OBYTES(vsi, es->tx_bytes);
5007         IXL_SET_IMCASTS(vsi, es->rx_multicast);
5008         IXL_SET_OMCASTS(vsi, es->tx_multicast);
5009
5010         IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
5011             nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
5012             nsd->rx_jabber);
5013         IXL_SET_OERRORS(vsi, es->tx_errors);
5014         IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
5015         IXL_SET_OQDROPS(vsi, tx_discards);
5016         IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
5017         IXL_SET_COLLISIONS(vsi, 0);
5018 }
5019
5020 /**
5021  * Reset all of the stats for the given pf
5022  **/
5023 void ixl_pf_reset_stats(struct ixl_pf *pf)
5024 {
5025         bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
5026         bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
5027         pf->stat_offsets_loaded = false;
5028 }
5029
5030 /**
5031  * Resets all stats of the given vsi
5032  **/
5033 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
5034 {
5035         bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
5036         bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
5037         vsi->stat_offsets_loaded = false;
5038 }
5039
5040 /**
5041  * Read and update a 48 bit stat from the hw
5042  *
5043  * Since the device stats are not reset at PFReset, they likely will not
5044  * be zeroed when the driver starts.  We'll save the first values read
5045  * and use them as offsets to be subtracted from the raw values in order
5046  * to report stats that count from zero.
5047  **/
5048 static void
5049 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
5050         bool offset_loaded, u64 *offset, u64 *stat)
5051 {
5052         u64 new_data;
5053
5054 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
5055         new_data = rd64(hw, loreg);
5056 #else
5057         /*
5058          * Use two rd32's instead of one rd64; FreeBSD versions before
5059          * 10 don't support 8 byte bus reads/writes.
5060          */
5061         new_data = rd32(hw, loreg);
5062         new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
5063 #endif
5064
5065         if (!offset_loaded)
5066                 *offset = new_data;
5067         if (new_data >= *offset)
5068                 *stat = new_data - *offset;
5069         else
5070                 *stat = (new_data + ((u64)1 << 48)) - *offset;
5071         *stat &= 0xFFFFFFFFFFFFULL;
5072 }
5073
5074 /**
5075  * Read and update a 32 bit stat from the hw
5076  **/
5077 static void
5078 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
5079         bool offset_loaded, u64 *offset, u64 *stat)
5080 {
5081         u32 new_data;
5082
5083         new_data = rd32(hw, reg);
5084         if (!offset_loaded)
5085                 *offset = new_data;
5086         if (new_data >= *offset)
5087                 *stat = (u32)(new_data - *offset);
5088         else
5089                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
5090 }
5091
5092 static void
5093 ixl_add_device_sysctls(struct ixl_pf *pf)
5094 {
5095         device_t dev = pf->dev;
5096
5097         /* Set up sysctls */
5098         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5099             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5100             OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
5101             pf, 0, ixl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC);
5102
5103         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5104             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5105             OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
5106             pf, 0, ixl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE);
5107
5108         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5109             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5110             OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
5111             pf, 0, ixl_current_speed, "A", "Current Port Speed");
5112
5113         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5114             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5115             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
5116             pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
5117
5118 #if 0
5119         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5120             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5121             OID_AUTO, "rx_itr", CTLFLAG_RW,
5122             &ixl_rx_itr, 0, "RX ITR");
5123
5124         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5125             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5126             OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
5127             &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
5128
5129         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5130             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5131             OID_AUTO, "tx_itr", CTLFLAG_RW,
5132             &ixl_tx_itr, 0, "TX ITR");
5133
5134         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5135             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5136             OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
5137             &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
5138 #endif
5139
5140 #ifdef IXL_DEBUG_SYSCTL
5141         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5142             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5143             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
5144             ixl_debug_info, "I", "Debug Information");
5145
5146         /* Shared-code debug message level */
5147         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5148             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5149             OID_AUTO, "debug_mask", CTLFLAG_RW,
5150             &pf->hw.debug_mask, 0, "Debug Message Level");
5151
5152         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5153             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5154             OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
5155             pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS);
5156
5157         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5158             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5159             OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
5160             pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
5161
5162         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5163             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5164             OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
5165             pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
5166
5167         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5168             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5169             OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
5170             pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
5171
5172         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5173             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5174             OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
5175             pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
5176
5177 #ifdef PCI_IOV
5178         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5179             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5180             OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
5181             0, "PF/VF Virtual Channel debug level");
5182 #endif
5183 #endif
5184 }
5185
5186 /*
5187 ** Set flow control using sysctl:
5188 **      0 - off
5189 **      1 - rx pause
5190 **      2 - tx pause
5191 **      3 - full
5192 */
5193 static int
5194 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
5195 {
5196         /*
5197          * TODO: ensure tx CRC by hardware should be enabled
5198          * if tx flow control is enabled.
5199          * ^ N/A for 40G ports
5200          */
5201         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5202         struct i40e_hw *hw = &pf->hw;
5203         device_t dev = pf->dev;
5204         int requested_fc, error = 0;
5205         enum i40e_status_code aq_error = 0;
5206         u8 fc_aq_err = 0;
5207
5208         /* Get request */
5209         requested_fc = pf->fc;
5210         error = sysctl_handle_int(oidp, &requested_fc, 0, req);
5211         if ((error) || (req->newptr == NULL))
5212                 return (error);
5213         if (requested_fc < 0 || requested_fc > 3) {
5214                 device_printf(dev,
5215                     "Invalid fc mode; valid modes are 0 through 3\n");
5216                 return (EINVAL);
5217         }
5218
5219         /* Set fc ability for port */
5220         hw->fc.requested_mode = requested_fc;
5221         aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
5222         if (aq_error) {
5223                 device_printf(dev,
5224                     "%s: Error setting new fc mode %d; fc_err %#x\n",
5225                     __func__, aq_error, fc_aq_err);
5226                 return (EIO);
5227         }
5228         pf->fc = requested_fc;
5229
5230         /* Get new link state */
5231         i40e_msec_delay(250);
5232         hw->phy.get_link_info = TRUE;
5233         i40e_get_link_status(hw, &pf->link_up);
5234
5235         return (0);
5236 }
5237
5238 static int
5239 ixl_current_speed(SYSCTL_HANDLER_ARGS)
5240 {
5241         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5242         struct i40e_hw *hw = &pf->hw;
5243         int error = 0, index = 0;
5244
5245         char *speeds[] = {
5246                 "Unknown",
5247                 "100M",
5248                 "1G",
5249                 "10G",
5250                 "40G",
5251                 "20G"
5252         };
5253
5254         ixl_update_link_status(pf);
5255
5256         switch (hw->phy.link_info.link_speed) {
5257         case I40E_LINK_SPEED_100MB:
5258                 index = 1;
5259                 break;
5260         case I40E_LINK_SPEED_1GB:
5261                 index = 2;
5262                 break;
5263         case I40E_LINK_SPEED_10GB:
5264                 index = 3;
5265                 break;
5266         case I40E_LINK_SPEED_40GB:
5267                 index = 4;
5268                 break;
5269         case I40E_LINK_SPEED_20GB:
5270                 index = 5;
5271                 break;
5272         case I40E_LINK_SPEED_UNKNOWN:
5273         default:
5274                 index = 0;
5275                 break;
5276         }
5277
5278         error = sysctl_handle_string(oidp, speeds[index],
5279             strlen(speeds[index]), req);
5280         return (error);
5281 }
5282
5283 static int
5284 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
5285 {
5286         struct i40e_hw *hw = &pf->hw;
5287         device_t dev = pf->dev;
5288         struct i40e_aq_get_phy_abilities_resp abilities;
5289         struct i40e_aq_set_phy_config config;
5290         enum i40e_status_code aq_error = 0;
5291
5292         /* Get current capability information */
5293         aq_error = i40e_aq_get_phy_capabilities(hw,
5294             FALSE, FALSE, &abilities, NULL);
5295         if (aq_error) {
5296                 device_printf(dev,
5297                     "%s: Error getting phy capabilities %d,"
5298                     " aq error: %d\n", __func__, aq_error,
5299                     hw->aq.asq_last_status);
5300                 return (EAGAIN);
5301         }
5302
5303         /* Prepare new config */
5304         bzero(&config, sizeof(config));
5305         config.phy_type = abilities.phy_type;
5306         config.abilities = abilities.abilities
5307             | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
5308         config.eee_capability = abilities.eee_capability;
5309         config.eeer = abilities.eeer_val;
5310         config.low_power_ctrl = abilities.d3_lpan;
5311         /* Translate into aq cmd link_speed */
5312         if (speeds & 0x10)
5313                 config.link_speed |= I40E_LINK_SPEED_40GB;
5314         if (speeds & 0x8)
5315                 config.link_speed |= I40E_LINK_SPEED_20GB;
5316         if (speeds & 0x4)
5317                 config.link_speed |= I40E_LINK_SPEED_10GB;
5318         if (speeds & 0x2)
5319                 config.link_speed |= I40E_LINK_SPEED_1GB;
5320         if (speeds & 0x1)
5321                 config.link_speed |= I40E_LINK_SPEED_100MB;
5322
5323         /* Do aq command & restart link */
5324         aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
5325         if (aq_error) {
5326                 device_printf(dev,
5327                     "%s: Error setting new phy config %d,"
5328                     " aq error: %d\n", __func__, aq_error,
5329                     hw->aq.asq_last_status);
5330                 return (EAGAIN);
5331         }
5332
5333         /*
5334         ** This seems a bit heavy handed, but we
5335         ** need to get a reinit on some devices
5336         */
5337         IXL_PF_LOCK(pf);
5338         ixl_stop_locked(pf);
5339         ixl_init_locked(pf);
5340         IXL_PF_UNLOCK(pf);
5341
5342         return (0);
5343 }
5344
5345 /*
5346 ** Control link advertise speed:
5347 **      Flags:
5348 **       0x1 - advertise 100 Mb
5349 **       0x2 - advertise 1G
5350 **       0x4 - advertise 10G
5351 **       0x8 - advertise 20G
5352 **      0x10 - advertise 40G
5353 **
5354 **      Set to 0 to disable link
5355 */
5356 static int
5357 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
5358 {
5359         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5360         struct i40e_hw *hw = &pf->hw;
5361         device_t dev = pf->dev;
5362         int requested_ls = 0;
5363         int error = 0;
5364
5365         /* Read in new mode */
5366         requested_ls = pf->advertised_speed;
5367         error = sysctl_handle_int(oidp, &requested_ls, 0, req);
5368         if ((error) || (req->newptr == NULL))
5369                 return (error);
5370         /* Check for sane value */
5371         if (requested_ls > 0x10) {
5372                 device_printf(dev, "Invalid advertised speed; "
5373                     "valid modes are 0x1 through 0x10\n");
5374                 return (EINVAL);
5375         }
5376         /* Then check for validity based on adapter type */
5377         switch (hw->device_id) {
5378         case I40E_DEV_ID_10G_BASE_T:
5379         case I40E_DEV_ID_10G_BASE_T4:
5380                 /* BaseT */
5381                 if (requested_ls & ~(0x7)) {
5382                         device_printf(dev,
5383                             "Only 100M/1G/10G speeds supported on this device.\n");
5384                         return (EINVAL);
5385                 }
5386                 break;
5387         case I40E_DEV_ID_20G_KR2:
5388         case I40E_DEV_ID_20G_KR2_A:
5389                 /* 20G */
5390                 if (requested_ls & ~(0xE)) {
5391                         device_printf(dev,
5392                             "Only 1G/10G/20G speeds supported on this device.\n");
5393                         return (EINVAL);
5394                 }
5395                 break;
5396         case I40E_DEV_ID_KX_B:
5397         case I40E_DEV_ID_QSFP_A:
5398         case I40E_DEV_ID_QSFP_B:
5399                 /* 40G */
5400                 if (requested_ls & ~(0x10)) {
5401                         device_printf(dev,
5402                             "Only 40G speeds supported on this device.\n");
5403                         return (EINVAL);
5404                 }
5405                 break;
5406         default:
5407                 /* 10G (1G) */
5408                 if (requested_ls & ~(0x6)) {
5409                         device_printf(dev,
5410                             "Only 1/10Gbs speeds are supported on this device.\n");
5411                         return (EINVAL);
5412                 }
5413                 break;
5414         }
5415
5416         /* Exit if no change */
5417         if (pf->advertised_speed == requested_ls)
5418                 return (0);
5419
5420         error = ixl_set_advertised_speeds(pf, requested_ls);
5421         if (error)
5422                 return (error);
5423
5424         pf->advertised_speed = requested_ls;
5425         ixl_update_link_status(pf);
5426         return (0);
5427 }
5428
5429 /*
5430 ** Get the width and transaction speed of
5431 ** the bus this adapter is plugged into.
5432 */
5433 static u16
5434 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
5435 {
5436         u16                     link;
5437         u32                     offset;
5438
5439         /* Get the PCI Express Capabilities offset */
5440         pci_find_cap(dev, PCIY_EXPRESS, &offset);
5441
5442         /* ...and read the Link Status Register */
5443         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
5444
5445         switch (link & I40E_PCI_LINK_WIDTH) {
5446         case I40E_PCI_LINK_WIDTH_1:
5447                 hw->bus.width = i40e_bus_width_pcie_x1;
5448                 break;
5449         case I40E_PCI_LINK_WIDTH_2:
5450                 hw->bus.width = i40e_bus_width_pcie_x2;
5451                 break;
5452         case I40E_PCI_LINK_WIDTH_4:
5453                 hw->bus.width = i40e_bus_width_pcie_x4;
5454                 break;
5455         case I40E_PCI_LINK_WIDTH_8:
5456                 hw->bus.width = i40e_bus_width_pcie_x8;
5457                 break;
5458         default:
5459                 hw->bus.width = i40e_bus_width_unknown;
5460                 break;
5461         }
5462
5463         switch (link & I40E_PCI_LINK_SPEED) {
5464         case I40E_PCI_LINK_SPEED_2500:
5465                 hw->bus.speed = i40e_bus_speed_2500;
5466                 break;
5467         case I40E_PCI_LINK_SPEED_5000:
5468                 hw->bus.speed = i40e_bus_speed_5000;
5469                 break;
5470         case I40E_PCI_LINK_SPEED_8000:
5471                 hw->bus.speed = i40e_bus_speed_8000;
5472                 break;
5473         default:
5474                 hw->bus.speed = i40e_bus_speed_unknown;
5475                 break;
5476         }
5477
5478         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
5479             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
5480             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
5481             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
5482             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
5483             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
5484             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
5485             ("Unknown"));
5486
5487         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
5488             (hw->bus.speed < i40e_bus_speed_8000)) {
5489                 device_printf(dev, "PCI-Express bandwidth available"
5490                     " for this device\n     may be insufficient for"
5491                     " optimal performance.\n");
5492                 device_printf(dev, "For expected performance a x8 "
5493                     "PCIE Gen3 slot is required.\n");
5494         }
5495
5496         return (link);
5497 }
5498
5499 static int
5500 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
5501 {
5502         struct ixl_pf   *pf = (struct ixl_pf *)arg1;
5503         struct i40e_hw  *hw = &pf->hw;
5504         struct sbuf     *sbuf;
5505
5506         sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5507         ixl_nvm_version_str(hw, sbuf);
5508         sbuf_finish(sbuf);
5509         sbuf_delete(sbuf);
5510
5511         return 0;
5512 }
5513
5514 #ifdef IXL_DEBUG
5515 static void
5516 ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma)
5517 {
5518         if ((nvma->command == I40E_NVM_READ) &&
5519             ((nvma->config & 0xFF) == 0xF) &&
5520             (((nvma->config & 0xF00) >> 8) == 0xF) &&
5521             (nvma->offset == 0) &&
5522             (nvma->data_size == 1)) {
5523                 // device_printf(dev, "- Get Driver Status Command\n");
5524         }
5525         else if (nvma->command == I40E_NVM_READ) {
5526
5527         }
5528         else {
5529                 switch (nvma->command) {
5530                 case 0xB:
5531                         device_printf(dev, "- command: I40E_NVM_READ\n");
5532                         break;
5533                 case 0xC:
5534                         device_printf(dev, "- command: I40E_NVM_WRITE\n");
5535                         break;
5536                 default:
5537                         device_printf(dev, "- command: unknown 0x%08x\n", nvma->command);
5538                         break;
5539                 }
5540
5541                 device_printf(dev, "- config (ptr)  : 0x%02x\n", nvma->config & 0xFF);
5542                 device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8);
5543                 device_printf(dev, "- offset : 0x%08x\n", nvma->offset);
5544                 device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size);
5545         }
5546 }
5547 #endif
5548
5549 static int
5550 ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd)
5551 {
5552         struct i40e_hw *hw = &pf->hw;
5553         struct i40e_nvm_access *nvma;
5554         device_t dev = pf->dev;
5555         enum i40e_status_code status = 0;
5556         int perrno;
5557
5558         DEBUGFUNC("ixl_handle_nvmupd_cmd");
5559
5560         /* Sanity checks */
5561         if (ifd->ifd_len < sizeof(struct i40e_nvm_access) ||
5562             ifd->ifd_data == NULL) {
5563                 device_printf(dev, "%s: incorrect ifdrv length or data pointer\n",
5564                     __func__);
5565                 device_printf(dev, "%s: ifdrv length: %lu, sizeof(struct i40e_nvm_access): %lu\n",
5566                     __func__, ifd->ifd_len, sizeof(struct i40e_nvm_access));
5567                 device_printf(dev, "%s: data pointer: %p\n", __func__,
5568                     ifd->ifd_data);
5569                 return (EINVAL);
5570         }
5571
5572         nvma = (struct i40e_nvm_access *)ifd->ifd_data;
5573
5574 #ifdef IXL_DEBUG
5575         ixl_print_nvm_cmd(dev, nvma);
5576 #endif
5577
5578         if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
5579                 int count = 0;
5580                 while (count++ < 100) {
5581                         i40e_msec_delay(100);
5582                         if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING))
5583                                 break;
5584                 }
5585         }
5586
5587         if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) {
5588                 IXL_PF_LOCK(pf);
5589                 status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno);
5590                 IXL_PF_UNLOCK(pf);
5591         } else {
5592                 perrno = -EBUSY;
5593         }
5594
5595         if (status)
5596                 device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n",
5597                     status, perrno);
5598
5599         /*
5600          * -EPERM is actually ERESTART, which the kernel interprets as it needing
5601          * to run this ioctl again. So use -EACCES for -EPERM instead.
5602          */
5603         if (perrno == -EPERM)
5604                 return (-EACCES);
5605         else
5606                 return (perrno);
5607 }
5608
5609 #ifdef IXL_DEBUG_SYSCTL
5610 static int
5611 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
5612 {
5613         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5614         struct i40e_hw *hw = &pf->hw;
5615         struct i40e_link_status link_status;
5616         char buf[512];
5617
5618         enum i40e_status_code aq_error = 0;
5619
5620         aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
5621         if (aq_error) {
5622                 printf("i40e_aq_get_link_info() error %d\n", aq_error);
5623                 return (EPERM);
5624         }
5625
5626         sprintf(buf, "\n"
5627             "PHY Type : %#04x\n"
5628             "Speed    : %#04x\n"
5629             "Link info: %#04x\n"
5630             "AN info  : %#04x\n"
5631             "Ext info : %#04x\n"
5632             "Max Frame: %d\n"
5633             "Pacing   : %#04x\n"
5634             "CRC En?  : %d",
5635             link_status.phy_type, link_status.link_speed,
5636             link_status.link_info, link_status.an_info,
5637             link_status.ext_info, link_status.max_frame_size,
5638             link_status.pacing, link_status.crc_enable);
5639
5640         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5641 }
5642
5643 static int
5644 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
5645 {
5646         struct ixl_pf           *pf = (struct ixl_pf *)arg1;
5647         struct i40e_hw          *hw = &pf->hw;
5648         char                    buf[512];
5649         enum i40e_status_code   aq_error = 0;
5650
5651         struct i40e_aq_get_phy_abilities_resp abilities;
5652
5653         aq_error = i40e_aq_get_phy_capabilities(hw,
5654             TRUE, FALSE, &abilities, NULL);
5655         if (aq_error) {
5656                 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
5657                 return (EPERM);
5658         }
5659
5660         sprintf(buf, "\n"
5661             "PHY Type : %#010x\n"
5662             "Speed    : %#04x\n"
5663             "Abilities: %#04x\n"
5664             "EEE cap  : %#06x\n"
5665             "EEER reg : %#010x\n"
5666             "D3 Lpan  : %#04x",
5667             abilities.phy_type, abilities.link_speed,
5668             abilities.abilities, abilities.eee_capability,
5669             abilities.eeer_val, abilities.d3_lpan);
5670
5671         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5672 }
5673
5674 static int
5675 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5676 {
5677         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5678         struct ixl_vsi *vsi = &pf->vsi;
5679         struct ixl_mac_filter *f;
5680         char *buf, *buf_i;
5681
5682         int error = 0;
5683         int ftl_len = 0;
5684         int ftl_counter = 0;
5685         int buf_len = 0;
5686         int entry_len = 42;
5687
5688         SLIST_FOREACH(f, &vsi->ftl, next) {
5689                 ftl_len++;
5690         }
5691
5692         if (ftl_len < 1) {
5693                 sysctl_handle_string(oidp, "(none)", 6, req);
5694                 return (0);
5695         }
5696
5697         buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5698         buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5699
5700         sprintf(buf_i++, "\n");
5701         SLIST_FOREACH(f, &vsi->ftl, next) {
5702                 sprintf(buf_i,
5703                     MAC_FORMAT ", vlan %4d, flags %#06x",
5704                     MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5705                 buf_i += entry_len;
5706                 /* don't print '\n' for last entry */
5707                 if (++ftl_counter != ftl_len) {
5708                         sprintf(buf_i, "\n");
5709                         buf_i++;
5710                 }
5711         }
5712
5713         error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5714         if (error)
5715                 printf("sysctl error: %d\n", error);
5716         free(buf, M_DEVBUF);
5717         return error;
5718 }
5719
5720 #define IXL_SW_RES_SIZE 0x14
5721 static int
5722 ixl_res_alloc_cmp(const void *a, const void *b)
5723 {
5724         const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5725         one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5726         two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5727
5728         return ((int)one->resource_type - (int)two->resource_type);
5729 }
5730
5731 /*
5732  * Longest string length: 25
5733  */
5734 static char *
5735 ixl_switch_res_type_string(u8 type)
5736 {
5737         static char * ixl_switch_res_type_strings[0x14] = {
5738                 "VEB",
5739                 "VSI",
5740                 "Perfect Match MAC address",
5741                 "S-tag",
5742                 "(Reserved)",
5743                 "Multicast hash entry",
5744                 "Unicast hash entry",
5745                 "VLAN",
5746                 "VSI List entry",
5747                 "(Reserved)",
5748                 "VLAN Statistic Pool",
5749                 "Mirror Rule",
5750                 "Queue Set",
5751                 "Inner VLAN Forward filter",
5752                 "(Reserved)",
5753                 "Inner MAC",
5754                 "IP",
5755                 "GRE/VN1 Key",
5756                 "VN2 Key",
5757                 "Tunneling Port"
5758         };
5759
5760         if (type < 0x14)
5761                 return ixl_switch_res_type_strings[type];
5762         else
5763                 return "(Reserved)";
5764 }
5765
5766 static int
5767 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5768 {
5769         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5770         struct i40e_hw *hw = &pf->hw;
5771         device_t dev = pf->dev;
5772         struct sbuf *buf;
5773         int error = 0;
5774
5775         u8 num_entries;
5776         struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5777
5778         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5779         if (!buf) {
5780                 device_printf(dev, "Could not allocate sbuf for output.\n");
5781                 return (ENOMEM);
5782         }
5783
5784         bzero(resp, sizeof(resp));
5785         error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5786                                 resp,
5787                                 IXL_SW_RES_SIZE,
5788                                 NULL);
5789         if (error) {
5790                 device_printf(dev,
5791                     "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5792                     __func__, error, hw->aq.asq_last_status);
5793                 sbuf_delete(buf);
5794                 return error;
5795         }
5796
5797         /* Sort entries by type for display */
5798         qsort(resp, num_entries,
5799             sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5800             &ixl_res_alloc_cmp);
5801
5802         sbuf_cat(buf, "\n");
5803         sbuf_printf(buf, "# of entries: %d\n", num_entries);
5804         sbuf_printf(buf,
5805             "                     Type | Guaranteed | Total | Used   | Un-allocated\n"
5806             "                          | (this)     | (all) | (this) | (all)       \n");
5807         for (int i = 0; i < num_entries; i++) {
5808                 sbuf_printf(buf,
5809                     "%25s | %10d   %5d   %6d   %12d",
5810                     ixl_switch_res_type_string(resp[i].resource_type),
5811                     resp[i].guaranteed,
5812                     resp[i].total,
5813                     resp[i].used,
5814                     resp[i].total_unalloced);
5815                 if (i < num_entries - 1)
5816                         sbuf_cat(buf, "\n");
5817         }
5818
5819         error = sbuf_finish(buf);
5820         if (error)
5821                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5822
5823         sbuf_delete(buf);
5824         return error;
5825 }
5826
5827 /*
5828 ** Caller must init and delete sbuf; this function will clear and
5829 ** finish it for caller.
5830 **
5831 ** XXX: Cannot use the SEID for this, since there is no longer a
5832 ** fixed mapping between SEID and element type.
5833 */
5834 static char *
5835 ixl_switch_element_string(struct sbuf *s,
5836     struct i40e_aqc_switch_config_element_resp *element)
5837 {
5838         sbuf_clear(s);
5839
5840         switch (element->element_type) {
5841         case I40E_AQ_SW_ELEM_TYPE_MAC:
5842                 sbuf_printf(s, "MAC %3d", element->element_info);
5843                 break;
5844         case I40E_AQ_SW_ELEM_TYPE_PF:
5845                 sbuf_printf(s, "PF  %3d", element->element_info);
5846                 break;
5847         case I40E_AQ_SW_ELEM_TYPE_VF:
5848                 sbuf_printf(s, "VF  %3d", element->element_info);
5849                 break;
5850         case I40E_AQ_SW_ELEM_TYPE_EMP:
5851                 sbuf_cat(s, "EMP");
5852                 break;
5853         case I40E_AQ_SW_ELEM_TYPE_BMC:
5854                 sbuf_cat(s, "BMC");
5855                 break;
5856         case I40E_AQ_SW_ELEM_TYPE_PV:
5857                 sbuf_cat(s, "PV");
5858                 break;
5859         case I40E_AQ_SW_ELEM_TYPE_VEB:
5860                 sbuf_cat(s, "VEB");
5861                 break;
5862         case I40E_AQ_SW_ELEM_TYPE_PA:
5863                 sbuf_cat(s, "PA");
5864                 break;
5865         case I40E_AQ_SW_ELEM_TYPE_VSI:
5866                 sbuf_printf(s, "VSI %3d", element->element_info);
5867                 break;
5868         default:
5869                 sbuf_cat(s, "?");
5870                 break;
5871         }
5872
5873         sbuf_finish(s);
5874         return sbuf_data(s);
5875 }
5876
5877 static int
5878 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5879 {
5880         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5881         struct i40e_hw *hw = &pf->hw;
5882         device_t dev = pf->dev;
5883         struct sbuf *buf;
5884         struct sbuf *nmbuf;
5885         int error = 0;
5886         u16 next = 0;
5887         u8 aq_buf[I40E_AQ_LARGE_BUF];
5888
5889         struct i40e_aqc_get_switch_config_resp *sw_config;
5890         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5891
5892         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5893         if (!buf) {
5894                 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5895                 return (ENOMEM);
5896         }
5897
5898         error = i40e_aq_get_switch_config(hw, sw_config,
5899             sizeof(aq_buf), &next, NULL);
5900         if (error) {
5901                 device_printf(dev,
5902                     "%s: aq_get_switch_config() error %d, aq error %d\n",
5903                     __func__, error, hw->aq.asq_last_status);
5904                 sbuf_delete(buf);
5905                 return error;
5906         }
5907         if (next)
5908                 device_printf(dev, "%s: TODO: get more config with SEID %d\n",
5909                     __func__, next);
5910
5911         nmbuf = sbuf_new_auto();
5912         if (!nmbuf) {
5913                 device_printf(dev, "Could not allocate sbuf for name output.\n");
5914                 sbuf_delete(buf);
5915                 return (ENOMEM);
5916         }
5917
5918         sbuf_cat(buf, "\n");
5919         // Assuming <= 255 elements in switch
5920         sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported);
5921         sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total);
5922         /* Exclude:
5923         ** Revision -- all elements are revision 1 for now
5924         */
5925         sbuf_printf(buf,
5926             "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5927             "                |          |          | (uplink)\n");
5928         for (int i = 0; i < sw_config->header.num_reported; i++) {
5929                 // "%4d (%8s) | %8s   %8s   %#8x",
5930                 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5931                 sbuf_cat(buf, " ");
5932                 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5933                     &sw_config->element[i]));
5934                 sbuf_cat(buf, " | ");
5935                 sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid);
5936                 sbuf_cat(buf, "   ");
5937                 sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid);
5938                 sbuf_cat(buf, "   ");
5939                 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5940                 if (i < sw_config->header.num_reported - 1)
5941                         sbuf_cat(buf, "\n");
5942         }
5943         sbuf_delete(nmbuf);
5944
5945         error = sbuf_finish(buf);
5946         if (error)
5947                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5948
5949         sbuf_delete(buf);
5950
5951         return (error);
5952 }
5953
5954 static int
5955 ixl_debug_info(SYSCTL_HANDLER_ARGS)
5956 {
5957         struct ixl_pf   *pf;
5958         int             error, input = 0;
5959
5960         error = sysctl_handle_int(oidp, &input, 0, req);
5961
5962         if (error || !req->newptr)
5963                 return (error);
5964
5965         if (input == 1) {
5966                 pf = (struct ixl_pf *)arg1;
5967                 ixl_print_debug_info(pf);
5968         }
5969
5970         return (error);
5971 }
5972
5973 static void
5974 ixl_print_debug_info(struct ixl_pf *pf)
5975 {
5976         struct i40e_hw          *hw = &pf->hw;
5977         struct ixl_vsi          *vsi = &pf->vsi;
5978         struct ixl_queue        *que = vsi->queues;
5979         struct rx_ring          *rxr = &que->rxr;
5980         struct tx_ring          *txr = &que->txr;
5981         u32                     reg;
5982
5983
5984         printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
5985         printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
5986         printf("RX next check = %x\n", rxr->next_check);
5987         printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
5988         printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
5989         printf("TX desc avail = %x\n", txr->avail);
5990
5991         reg = rd32(hw, I40E_GLV_GORCL(0xc));
5992          printf("RX Bytes = %x\n", reg);
5993         reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
5994          printf("Port RX Bytes = %x\n", reg);
5995         reg = rd32(hw, I40E_GLV_RDPC(0xc));
5996          printf("RX discard = %x\n", reg);
5997         reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
5998          printf("Port RX discard = %x\n", reg);
5999
6000         reg = rd32(hw, I40E_GLV_TEPC(0xc));
6001          printf("TX errors = %x\n", reg);
6002         reg = rd32(hw, I40E_GLV_GOTCL(0xc));
6003          printf("TX Bytes = %x\n", reg);
6004
6005         reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
6006          printf("RX undersize = %x\n", reg);
6007         reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
6008          printf("RX fragments = %x\n", reg);
6009         reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
6010          printf("RX oversize = %x\n", reg);
6011         reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
6012          printf("RX length error = %x\n", reg);
6013         reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
6014          printf("mac remote fault = %x\n", reg);
6015         reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
6016          printf("mac local fault = %x\n", reg);
6017 }
6018
6019 #endif /* IXL_DEBUG_SYSCTL */
6020
6021 #ifdef PCI_IOV
6022 static int
6023 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6024 {
6025         struct i40e_hw *hw;
6026         struct ixl_vsi *vsi;
6027         struct i40e_vsi_context vsi_ctx;
6028         int i;
6029         uint16_t first_queue;
6030         enum i40e_status_code code;
6031
6032         hw = &pf->hw;
6033         vsi = &pf->vsi;
6034
6035         vsi_ctx.pf_num = hw->pf_id;
6036         vsi_ctx.uplink_seid = pf->veb_seid;
6037         vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
6038         vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6039         vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
6040
6041         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6042
6043         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6044         vsi_ctx.info.switch_id = htole16(0);
6045
6046         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
6047         vsi_ctx.info.sec_flags = 0;
6048         if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
6049                 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
6050
6051         /* TODO: If a port VLAN is set, then this needs to be changed */
6052         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6053         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6054             I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
6055
6056         vsi_ctx.info.valid_sections |=
6057             htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
6058         vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
6059         first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
6060         for (i = 0; i < IXLV_MAX_QUEUES; i++)
6061                 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
6062         for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
6063                 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
6064
6065         vsi_ctx.info.tc_mapping[0] = htole16(
6066             (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6067             (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
6068
6069         code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
6070         if (code != I40E_SUCCESS)
6071                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6072         vf->vsi.seid = vsi_ctx.seid;
6073         vf->vsi.vsi_num = vsi_ctx.vsi_number;
6074         vf->vsi.first_queue = first_queue;
6075         vf->vsi.num_queues = IXLV_MAX_QUEUES;
6076
6077         code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
6078         if (code != I40E_SUCCESS)
6079                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6080
6081         code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
6082         if (code != I40E_SUCCESS) {
6083                 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
6084                     ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6085                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6086         }
6087
6088         memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
6089         return (0);
6090 }
6091
6092 static int
6093 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6094 {
6095         struct i40e_hw *hw;
6096         int error;
6097
6098         hw = &pf->hw;
6099
6100         error = ixl_vf_alloc_vsi(pf, vf);
6101         if (error != 0)
6102                 return (error);
6103
6104         vf->vsi.hw_filters_add = 0;
6105         vf->vsi.hw_filters_del = 0;
6106         ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
6107         ixl_reconfigure_filters(&vf->vsi);
6108
6109         return (0);
6110 }
6111
6112 static void
6113 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
6114     uint32_t val)
6115 {
6116         uint32_t qtable;
6117         int index, shift;
6118
6119         /*
6120          * Two queues are mapped in a single register, so we have to do some
6121          * gymnastics to convert the queue number into a register index and
6122          * shift.
6123          */
6124         index = qnum / 2;
6125         shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
6126
6127         qtable = i40e_read_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
6128         qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
6129         qtable |= val << shift;
6130         i40e_write_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
6131 }
6132
6133 static void
6134 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
6135 {
6136         struct i40e_hw *hw;
6137         uint32_t qtable;
6138         int i;
6139
6140         hw = &pf->hw;
6141
6142         /*
6143          * Contiguous mappings aren't actually supported by the hardware,
6144          * so we have to use non-contiguous mappings.
6145          */
6146         i40e_write_rx_ctl(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
6147              I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
6148
6149         wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
6150             I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
6151
6152         for (i = 0; i < vf->vsi.num_queues; i++) {
6153                 qtable = (vf->vsi.first_queue + i) <<
6154                     I40E_VPLAN_QTABLE_QINDEX_SHIFT;
6155
6156                 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
6157         }
6158
6159         /* Map queues allocated to VF to its VSI. */
6160         for (i = 0; i < vf->vsi.num_queues; i++)
6161                 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
6162
6163         /* Set rest of VSI queues as unused. */
6164         for (; i < IXL_MAX_VSI_QUEUES; i++)
6165                 ixl_vf_map_vsi_queue(hw, vf, i,
6166                     I40E_VSILAN_QTABLE_QINDEX_0_MASK);
6167
6168         ixl_flush(hw);
6169 }
6170
6171 static void
6172 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
6173 {
6174         struct i40e_hw *hw;
6175
6176         hw = &pf->hw;
6177
6178         if (vsi->seid == 0)
6179                 return;
6180
6181         i40e_aq_delete_element(hw, vsi->seid, NULL);
6182 }
6183
6184 static void
6185 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
6186 {
6187
6188         wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
6189         ixl_flush(hw);
6190 }
6191
6192 static void
6193 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
6194 {
6195
6196         wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
6197             I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
6198         ixl_flush(hw);
6199 }
6200
6201 static void
6202 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
6203 {
6204         struct i40e_hw *hw;
6205         uint32_t vfint_reg, vpint_reg;
6206         int i;
6207
6208         hw = &pf->hw;
6209
6210         ixl_vf_vsi_release(pf, &vf->vsi);
6211
6212         /* Index 0 has a special register. */
6213         ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
6214
6215         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6216                 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
6217                 ixl_vf_disable_queue_intr(hw, vfint_reg);
6218         }
6219
6220         /* Index 0 has a special register. */
6221         ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
6222
6223         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6224                 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
6225                 ixl_vf_unregister_intr(hw, vpint_reg);
6226         }
6227
6228         vf->vsi.num_queues = 0;
6229 }
6230
6231 static int
6232 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
6233 {
6234         struct i40e_hw *hw;
6235         int i;
6236         uint16_t global_vf_num;
6237         uint32_t ciad;
6238
6239         hw = &pf->hw;
6240         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6241
6242         wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
6243              (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
6244         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6245                 ciad = rd32(hw, I40E_PF_PCI_CIAD);
6246                 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
6247                         return (0);
6248                 DELAY(1);
6249         }
6250
6251         return (ETIMEDOUT);
6252 }
6253
6254 static void
6255 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6256 {
6257         struct i40e_hw *hw;
6258         uint32_t vfrtrig;
6259
6260         hw = &pf->hw;
6261
6262         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6263         vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6264         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6265         ixl_flush(hw);
6266
6267         ixl_reinit_vf(pf, vf);
6268 }
6269
6270 static void
6271 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6272 {
6273         struct i40e_hw *hw;
6274         uint32_t vfrstat, vfrtrig;
6275         int i, error;
6276
6277         hw = &pf->hw;
6278
6279         error = ixl_flush_pcie(pf, vf);
6280         if (error != 0)
6281                 device_printf(pf->dev,
6282                     "Timed out waiting for PCIe activity to stop on VF-%d\n",
6283                     vf->vf_num);
6284
6285         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6286                 DELAY(10);
6287
6288                 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
6289                 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
6290                         break;
6291         }
6292
6293         if (i == IXL_VF_RESET_TIMEOUT)
6294                 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
6295
6296         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
6297
6298         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6299         vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6300         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6301
6302         if (vf->vsi.seid != 0)
6303                 ixl_disable_rings(&vf->vsi);
6304
6305         ixl_vf_release_resources(pf, vf);
6306         ixl_vf_setup_vsi(pf, vf);
6307         ixl_vf_map_queues(pf, vf);
6308
6309         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
6310         ixl_flush(hw);
6311 }
6312
6313 static const char *
6314 ixl_vc_opcode_str(uint16_t op)
6315 {
6316
6317         switch (op) {
6318         case I40E_VIRTCHNL_OP_VERSION:
6319                 return ("VERSION");
6320         case I40E_VIRTCHNL_OP_RESET_VF:
6321                 return ("RESET_VF");
6322         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6323                 return ("GET_VF_RESOURCES");
6324         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6325                 return ("CONFIG_TX_QUEUE");
6326         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6327                 return ("CONFIG_RX_QUEUE");
6328         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6329                 return ("CONFIG_VSI_QUEUES");
6330         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6331                 return ("CONFIG_IRQ_MAP");
6332         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6333                 return ("ENABLE_QUEUES");
6334         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6335                 return ("DISABLE_QUEUES");
6336         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6337                 return ("ADD_ETHER_ADDRESS");
6338         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6339                 return ("DEL_ETHER_ADDRESS");
6340         case I40E_VIRTCHNL_OP_ADD_VLAN:
6341                 return ("ADD_VLAN");
6342         case I40E_VIRTCHNL_OP_DEL_VLAN:
6343                 return ("DEL_VLAN");
6344         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6345                 return ("CONFIG_PROMISCUOUS_MODE");
6346         case I40E_VIRTCHNL_OP_GET_STATS:
6347                 return ("GET_STATS");
6348         case I40E_VIRTCHNL_OP_FCOE:
6349                 return ("FCOE");
6350         case I40E_VIRTCHNL_OP_EVENT:
6351                 return ("EVENT");
6352         default:
6353                 return ("UNKNOWN");
6354         }
6355 }
6356
6357 static int
6358 ixl_vc_opcode_level(uint16_t opcode)
6359 {
6360         switch (opcode) {
6361         case I40E_VIRTCHNL_OP_GET_STATS:
6362                 return (10);
6363         default:
6364                 return (5);
6365         }
6366 }
6367
6368 static void
6369 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6370     enum i40e_status_code status, void *msg, uint16_t len)
6371 {
6372         struct i40e_hw *hw;
6373         int global_vf_id;
6374
6375         hw = &pf->hw;
6376         global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
6377
6378         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
6379             "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
6380             ixl_vc_opcode_str(op), op, status, vf->vf_num);
6381
6382         i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
6383 }
6384
6385 static void
6386 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
6387 {
6388
6389         ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
6390 }
6391
6392 static void
6393 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6394     enum i40e_status_code status, const char *file, int line)
6395 {
6396
6397         I40E_VC_DEBUG(pf, 1,
6398             "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
6399             ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
6400         ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
6401 }
6402
6403 static void
6404 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6405     uint16_t msg_size)
6406 {
6407         struct i40e_virtchnl_version_info reply;
6408
6409         if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
6410                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
6411                     I40E_ERR_PARAM);
6412                 return;
6413         }
6414
6415         vf->version = ((struct i40e_virtchnl_version_info *)msg)->minor;
6416
6417         reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
6418         reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
6419         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
6420             sizeof(reply));
6421 }
6422
6423 static void
6424 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6425     uint16_t msg_size)
6426 {
6427
6428         if (msg_size != 0) {
6429                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
6430                     I40E_ERR_PARAM);
6431                 return;
6432         }
6433
6434         ixl_reset_vf(pf, vf);
6435
6436         /* No response to a reset message. */
6437 }
6438
6439 static void
6440 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6441     uint16_t msg_size)
6442 {
6443         struct i40e_virtchnl_vf_resource reply;
6444
6445         if ((vf->version == 0 && msg_size != 0) ||
6446             (vf->version == 1 && msg_size != 4)) {
6447                 device_printf(pf->dev, "Invalid GET_VF_RESOURCES message size,"
6448                     " for VF version %d.%d\n", I40E_VIRTCHNL_VERSION_MAJOR,
6449                     vf->version);
6450                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6451                     I40E_ERR_PARAM);
6452                 return;
6453         }
6454
6455         bzero(&reply, sizeof(reply));
6456
6457         if (vf->version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS)
6458                 reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
6459                                          I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
6460                                          I40E_VIRTCHNL_VF_OFFLOAD_VLAN;
6461         else
6462                 reply.vf_offload_flags = *(u32 *)msg;
6463
6464         reply.num_vsis = 1;
6465         reply.num_queue_pairs = vf->vsi.num_queues;
6466         reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
6467         reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
6468         reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
6469         reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
6470         memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
6471
6472         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6473             I40E_SUCCESS, &reply, sizeof(reply));
6474 }
6475
6476 static int
6477 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6478     struct i40e_virtchnl_txq_info *info)
6479 {
6480         struct i40e_hw *hw;
6481         struct i40e_hmc_obj_txq txq;
6482         uint16_t global_queue_num, global_vf_num;
6483         enum i40e_status_code status;
6484         uint32_t qtx_ctl;
6485
6486         hw = &pf->hw;
6487         global_queue_num = vf->vsi.first_queue + info->queue_id;
6488         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6489         bzero(&txq, sizeof(txq));
6490
6491         status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
6492         if (status != I40E_SUCCESS)
6493                 return (EINVAL);
6494
6495         txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
6496
6497         txq.head_wb_ena = info->headwb_enabled;
6498         txq.head_wb_addr = info->dma_headwb_addr;
6499         txq.qlen = info->ring_len;
6500         txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
6501         txq.rdylist_act = 0;
6502
6503         status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
6504         if (status != I40E_SUCCESS)
6505                 return (EINVAL);
6506
6507         qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
6508             (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
6509             (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
6510         wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
6511         ixl_flush(hw);
6512
6513         return (0);
6514 }
6515
6516 static int
6517 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6518     struct i40e_virtchnl_rxq_info *info)
6519 {
6520         struct i40e_hw *hw;
6521         struct i40e_hmc_obj_rxq rxq;
6522         uint16_t global_queue_num;
6523         enum i40e_status_code status;
6524
6525         hw = &pf->hw;
6526         global_queue_num = vf->vsi.first_queue + info->queue_id;
6527         bzero(&rxq, sizeof(rxq));
6528
6529         if (info->databuffer_size > IXL_VF_MAX_BUFFER)
6530                 return (EINVAL);
6531
6532         if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
6533             info->max_pkt_size < ETHER_MIN_LEN)
6534                 return (EINVAL);
6535
6536         if (info->splithdr_enabled) {
6537                 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
6538                         return (EINVAL);
6539
6540                 rxq.hsplit_0 = info->rx_split_pos &
6541                     (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
6542                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
6543                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
6544                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
6545                 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
6546
6547                 rxq.dtype = 2;
6548         }
6549
6550         status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
6551         if (status != I40E_SUCCESS)
6552                 return (EINVAL);
6553
6554         rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
6555         rxq.qlen = info->ring_len;
6556
6557         rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
6558
6559         rxq.dsize = 1;
6560         rxq.crcstrip = 1;
6561         rxq.l2tsel = 1;
6562
6563         rxq.rxmax = info->max_pkt_size;
6564         rxq.tphrdesc_ena = 1;
6565         rxq.tphwdesc_ena = 1;
6566         rxq.tphdata_ena = 1;
6567         rxq.tphhead_ena = 1;
6568         rxq.lrxqthresh = 2;
6569         rxq.prefena = 1;
6570
6571         status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
6572         if (status != I40E_SUCCESS)
6573                 return (EINVAL);
6574
6575         return (0);
6576 }
6577
6578 static void
6579 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6580     uint16_t msg_size)
6581 {
6582         struct i40e_virtchnl_vsi_queue_config_info *info;
6583         struct i40e_virtchnl_queue_pair_info *pair;
6584         int i;
6585
6586         if (msg_size < sizeof(*info)) {
6587                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6588                     I40E_ERR_PARAM);
6589                 return;
6590         }
6591
6592         info = msg;
6593         if (info->num_queue_pairs == 0) {
6594                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6595                     I40E_ERR_PARAM);
6596                 return;
6597         }
6598
6599         if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
6600                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6601                     I40E_ERR_PARAM);
6602                 return;
6603         }
6604
6605         if (info->vsi_id != vf->vsi.vsi_num) {
6606                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6607                     I40E_ERR_PARAM);
6608                 return;
6609         }
6610
6611         for (i = 0; i < info->num_queue_pairs; i++) {
6612                 pair = &info->qpair[i];
6613
6614                 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
6615                     pair->rxq.vsi_id != vf->vsi.vsi_num ||
6616                     pair->txq.queue_id != pair->rxq.queue_id ||
6617                     pair->txq.queue_id >= vf->vsi.num_queues) {
6618
6619                         i40e_send_vf_nack(pf, vf,
6620                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6621                         return;
6622                 }
6623
6624                 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
6625                         i40e_send_vf_nack(pf, vf,
6626                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6627                         return;
6628                 }
6629
6630                 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
6631                         i40e_send_vf_nack(pf, vf,
6632                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6633                         return;
6634                 }
6635         }
6636
6637         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
6638 }
6639
6640 static void
6641 ixl_vf_set_qctl(struct ixl_pf *pf,
6642     const struct i40e_virtchnl_vector_map *vector,
6643     enum i40e_queue_type cur_type, uint16_t cur_queue,
6644     enum i40e_queue_type *last_type, uint16_t *last_queue)
6645 {
6646         uint32_t offset, qctl;
6647         uint16_t itr_indx;
6648
6649         if (cur_type == I40E_QUEUE_TYPE_RX) {
6650                 offset = I40E_QINT_RQCTL(cur_queue);
6651                 itr_indx = vector->rxitr_idx;
6652         } else {
6653                 offset = I40E_QINT_TQCTL(cur_queue);
6654                 itr_indx = vector->txitr_idx;
6655         }
6656
6657         qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
6658             (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
6659             (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
6660             I40E_QINT_RQCTL_CAUSE_ENA_MASK |
6661             (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
6662
6663         wr32(&pf->hw, offset, qctl);
6664
6665         *last_type = cur_type;
6666         *last_queue = cur_queue;
6667 }
6668
6669 static void
6670 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
6671     const struct i40e_virtchnl_vector_map *vector)
6672 {
6673         struct i40e_hw *hw;
6674         u_int qindex;
6675         enum i40e_queue_type type, last_type;
6676         uint32_t lnklst_reg;
6677         uint16_t rxq_map, txq_map, cur_queue, last_queue;
6678
6679         hw = &pf->hw;
6680
6681         rxq_map = vector->rxq_map;
6682         txq_map = vector->txq_map;
6683
6684         last_queue = IXL_END_OF_INTR_LNKLST;
6685         last_type = I40E_QUEUE_TYPE_RX;
6686
6687         /*
6688          * The datasheet says to optimize performance, RX queues and TX queues
6689          * should be interleaved in the interrupt linked list, so we process
6690          * both at once here.
6691          */
6692         while ((rxq_map != 0) || (txq_map != 0)) {
6693                 if (txq_map != 0) {
6694                         qindex = ffs(txq_map) - 1;
6695                         type = I40E_QUEUE_TYPE_TX;
6696                         cur_queue = vf->vsi.first_queue + qindex;
6697                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
6698                             &last_type, &last_queue);
6699                         txq_map &= ~(1 << qindex);
6700                 }
6701
6702                 if (rxq_map != 0) {
6703                         qindex = ffs(rxq_map) - 1;
6704                         type = I40E_QUEUE_TYPE_RX;
6705                         cur_queue = vf->vsi.first_queue + qindex;
6706                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
6707                             &last_type, &last_queue);
6708                         rxq_map &= ~(1 << qindex);
6709                 }
6710         }
6711
6712         if (vector->vector_id == 0)
6713                 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
6714         else
6715                 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
6716                     vf->vf_num);
6717         wr32(hw, lnklst_reg,
6718             (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
6719             (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
6720
6721         ixl_flush(hw);
6722 }
6723
6724 static void
6725 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6726     uint16_t msg_size)
6727 {
6728         struct i40e_virtchnl_irq_map_info *map;
6729         struct i40e_virtchnl_vector_map *vector;
6730         struct i40e_hw *hw;
6731         int i, largest_txq, largest_rxq;
6732
6733         hw = &pf->hw;
6734
6735         if (msg_size < sizeof(*map)) {
6736                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6737                     I40E_ERR_PARAM);
6738                 return;
6739         }
6740
6741         map = msg;
6742         if (map->num_vectors == 0) {
6743                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6744                     I40E_ERR_PARAM);
6745                 return;
6746         }
6747
6748         if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
6749                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6750                     I40E_ERR_PARAM);
6751                 return;
6752         }
6753
6754         for (i = 0; i < map->num_vectors; i++) {
6755                 vector = &map->vecmap[i];
6756
6757                 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
6758                     vector->vsi_id != vf->vsi.vsi_num) {
6759                         i40e_send_vf_nack(pf, vf,
6760                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
6761                         return;
6762                 }
6763
6764                 if (vector->rxq_map != 0) {
6765                         largest_rxq = fls(vector->rxq_map) - 1;
6766                         if (largest_rxq >= vf->vsi.num_queues) {
6767                                 i40e_send_vf_nack(pf, vf,
6768                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6769                                     I40E_ERR_PARAM);
6770                                 return;
6771                         }
6772                 }
6773
6774                 if (vector->txq_map != 0) {
6775                         largest_txq = fls(vector->txq_map) - 1;
6776                         if (largest_txq >= vf->vsi.num_queues) {
6777                                 i40e_send_vf_nack(pf, vf,
6778                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6779                                     I40E_ERR_PARAM);
6780                                 return;
6781                         }
6782                 }
6783
6784                 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6785                     vector->txitr_idx > IXL_MAX_ITR_IDX) {
6786                         i40e_send_vf_nack(pf, vf,
6787                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6788                             I40E_ERR_PARAM);
6789                         return;
6790                 }
6791
6792                 ixl_vf_config_vector(pf, vf, vector);
6793         }
6794
6795         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6796 }
6797
6798 static void
6799 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6800     uint16_t msg_size)
6801 {
6802         struct i40e_virtchnl_queue_select *select;
6803         int error;
6804
6805         if (msg_size != sizeof(*select)) {
6806                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6807                     I40E_ERR_PARAM);
6808                 return;
6809         }
6810
6811         select = msg;
6812         if (select->vsi_id != vf->vsi.vsi_num ||
6813             select->rx_queues == 0 || select->tx_queues == 0) {
6814                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6815                     I40E_ERR_PARAM);
6816                 return;
6817         }
6818
6819         error = ixl_enable_rings(&vf->vsi);
6820         if (error) {
6821                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6822                     I40E_ERR_TIMEOUT);
6823                 return;
6824         }
6825
6826         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6827 }
6828
6829 static void
6830 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6831     void *msg, uint16_t msg_size)
6832 {
6833         struct i40e_virtchnl_queue_select *select;
6834         int error;
6835
6836         if (msg_size != sizeof(*select)) {
6837                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6838                     I40E_ERR_PARAM);
6839                 return;
6840         }
6841
6842         select = msg;
6843         if (select->vsi_id != vf->vsi.vsi_num ||
6844             select->rx_queues == 0 || select->tx_queues == 0) {
6845                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6846                     I40E_ERR_PARAM);
6847                 return;
6848         }
6849
6850         error = ixl_disable_rings(&vf->vsi);
6851         if (error) {
6852                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6853                     I40E_ERR_TIMEOUT);
6854                 return;
6855         }
6856
6857         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6858 }
6859
6860 static boolean_t
6861 ixl_zero_mac(const uint8_t *addr)
6862 {
6863         uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6864
6865         return (cmp_etheraddr(addr, zero));
6866 }
6867
6868 static boolean_t
6869 ixl_bcast_mac(const uint8_t *addr)
6870 {
6871
6872         return (cmp_etheraddr(addr, ixl_bcast_addr));
6873 }
6874
6875 static int
6876 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6877 {
6878
6879         if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6880                 return (EINVAL);
6881
6882         /*
6883          * If the VF is not allowed to change its MAC address, don't let it
6884          * set a MAC filter for an address that is not a multicast address and
6885          * is not its assigned MAC.
6886          */
6887         if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6888             !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6889                 return (EPERM);
6890
6891         return (0);
6892 }
6893
6894 static void
6895 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6896     uint16_t msg_size)
6897 {
6898         struct i40e_virtchnl_ether_addr_list *addr_list;
6899         struct i40e_virtchnl_ether_addr *addr;
6900         struct ixl_vsi *vsi;
6901         int i;
6902         size_t expected_size;
6903
6904         vsi = &vf->vsi;
6905
6906         if (msg_size < sizeof(*addr_list)) {
6907                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6908                     I40E_ERR_PARAM);
6909                 return;
6910         }
6911
6912         addr_list = msg;
6913         expected_size = sizeof(*addr_list) +
6914             addr_list->num_elements * sizeof(*addr);
6915
6916         if (addr_list->num_elements == 0 ||
6917             addr_list->vsi_id != vsi->vsi_num ||
6918             msg_size != expected_size) {
6919                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6920                     I40E_ERR_PARAM);
6921                 return;
6922         }
6923
6924         for (i = 0; i < addr_list->num_elements; i++) {
6925                 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6926                         i40e_send_vf_nack(pf, vf,
6927                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6928                         return;
6929                 }
6930         }
6931
6932         for (i = 0; i < addr_list->num_elements; i++) {
6933                 addr = &addr_list->list[i];
6934                 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6935         }
6936
6937         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6938 }
6939
6940 static void
6941 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6942     uint16_t msg_size)
6943 {
6944         struct i40e_virtchnl_ether_addr_list *addr_list;
6945         struct i40e_virtchnl_ether_addr *addr;
6946         size_t expected_size;
6947         int i;
6948
6949         if (msg_size < sizeof(*addr_list)) {
6950                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6951                     I40E_ERR_PARAM);
6952                 return;
6953         }
6954
6955         addr_list = msg;
6956         expected_size = sizeof(*addr_list) +
6957             addr_list->num_elements * sizeof(*addr);
6958
6959         if (addr_list->num_elements == 0 ||
6960             addr_list->vsi_id != vf->vsi.vsi_num ||
6961             msg_size != expected_size) {
6962                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6963                     I40E_ERR_PARAM);
6964                 return;
6965         }
6966
6967         for (i = 0; i < addr_list->num_elements; i++) {
6968                 addr = &addr_list->list[i];
6969                 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6970                         i40e_send_vf_nack(pf, vf,
6971                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6972                         return;
6973                 }
6974         }
6975
6976         for (i = 0; i < addr_list->num_elements; i++) {
6977                 addr = &addr_list->list[i];
6978                 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6979         }
6980
6981         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6982 }
6983
6984 static enum i40e_status_code
6985 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6986 {
6987         struct i40e_vsi_context vsi_ctx;
6988
6989         vsi_ctx.seid = vf->vsi.seid;
6990
6991         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6992         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6993         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6994             I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6995         return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6996 }
6997
6998 static void
6999 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7000     uint16_t msg_size)
7001 {
7002         struct i40e_virtchnl_vlan_filter_list *filter_list;
7003         enum i40e_status_code code;
7004         size_t expected_size;
7005         int i;
7006
7007         if (msg_size < sizeof(*filter_list)) {
7008                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7009                     I40E_ERR_PARAM);
7010                 return;
7011         }
7012
7013         filter_list = msg;
7014         expected_size = sizeof(*filter_list) +
7015             filter_list->num_elements * sizeof(uint16_t);
7016         if (filter_list->num_elements == 0 ||
7017             filter_list->vsi_id != vf->vsi.vsi_num ||
7018             msg_size != expected_size) {
7019                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7020                     I40E_ERR_PARAM);
7021                 return;
7022         }
7023
7024         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7025                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7026                     I40E_ERR_PARAM);
7027                 return;
7028         }
7029
7030         for (i = 0; i < filter_list->num_elements; i++) {
7031                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7032                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7033                             I40E_ERR_PARAM);
7034                         return;
7035                 }
7036         }
7037
7038         code = ixl_vf_enable_vlan_strip(pf, vf);
7039         if (code != I40E_SUCCESS) {
7040                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7041                     I40E_ERR_PARAM);
7042         }
7043
7044         for (i = 0; i < filter_list->num_elements; i++)
7045                 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7046
7047         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
7048 }
7049
7050 static void
7051 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7052     uint16_t msg_size)
7053 {
7054         struct i40e_virtchnl_vlan_filter_list *filter_list;
7055         int i;
7056         size_t expected_size;
7057
7058         if (msg_size < sizeof(*filter_list)) {
7059                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7060                     I40E_ERR_PARAM);
7061                 return;
7062         }
7063
7064         filter_list = msg;
7065         expected_size = sizeof(*filter_list) +
7066             filter_list->num_elements * sizeof(uint16_t);
7067         if (filter_list->num_elements == 0 ||
7068             filter_list->vsi_id != vf->vsi.vsi_num ||
7069             msg_size != expected_size) {
7070                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7071                     I40E_ERR_PARAM);
7072                 return;
7073         }
7074
7075         for (i = 0; i < filter_list->num_elements; i++) {
7076                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7077                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7078                             I40E_ERR_PARAM);
7079                         return;
7080                 }
7081         }
7082
7083         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7084                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7085                     I40E_ERR_PARAM);
7086                 return;
7087         }
7088
7089         for (i = 0; i < filter_list->num_elements; i++)
7090                 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7091
7092         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
7093 }
7094
7095 static void
7096 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
7097     void *msg, uint16_t msg_size)
7098 {
7099         struct i40e_virtchnl_promisc_info *info;
7100         enum i40e_status_code code;
7101
7102         if (msg_size != sizeof(*info)) {
7103                 i40e_send_vf_nack(pf, vf,
7104                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7105                 return;
7106         }
7107
7108         if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) {
7109                 i40e_send_vf_nack(pf, vf,
7110                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7111                 return;
7112         }
7113
7114         info = msg;
7115         if (info->vsi_id != vf->vsi.vsi_num) {
7116                 i40e_send_vf_nack(pf, vf,
7117                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7118                 return;
7119         }
7120
7121         code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
7122             info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
7123         if (code != I40E_SUCCESS) {
7124                 i40e_send_vf_nack(pf, vf,
7125                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7126                 return;
7127         }
7128
7129         code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
7130             info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
7131         if (code != I40E_SUCCESS) {
7132                 i40e_send_vf_nack(pf, vf,
7133                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7134                 return;
7135         }
7136
7137         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
7138 }
7139
7140 static void
7141 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7142     uint16_t msg_size)
7143 {
7144         struct i40e_virtchnl_queue_select *queue;
7145
7146         if (msg_size != sizeof(*queue)) {
7147                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7148                     I40E_ERR_PARAM);
7149                 return;
7150         }
7151
7152         queue = msg;
7153         if (queue->vsi_id != vf->vsi.vsi_num) {
7154                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7155                     I40E_ERR_PARAM);
7156                 return;
7157         }
7158
7159         ixl_update_eth_stats(&vf->vsi);
7160
7161         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7162             I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
7163 }
7164
7165 static void
7166 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
7167 {
7168         struct ixl_vf *vf;
7169         void *msg;
7170         uint16_t vf_num, msg_size;
7171         uint32_t opcode;
7172
7173         vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
7174         opcode = le32toh(event->desc.cookie_high);
7175
7176         if (vf_num >= pf->num_vfs) {
7177                 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
7178                 return;
7179         }
7180
7181         vf = &pf->vfs[vf_num];
7182         msg = event->msg_buf;
7183         msg_size = event->msg_len;
7184
7185         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
7186             "Got msg %s(%d) from VF-%d of size %d\n",
7187             ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
7188
7189         switch (opcode) {
7190         case I40E_VIRTCHNL_OP_VERSION:
7191                 ixl_vf_version_msg(pf, vf, msg, msg_size);
7192                 break;
7193         case I40E_VIRTCHNL_OP_RESET_VF:
7194                 ixl_vf_reset_msg(pf, vf, msg, msg_size);
7195                 break;
7196         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
7197                 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
7198                 break;
7199         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
7200                 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
7201                 break;
7202         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
7203                 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
7204                 break;
7205         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
7206                 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
7207                 break;
7208         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
7209                 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
7210                 break;
7211         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
7212                 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
7213                 break;
7214         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
7215                 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
7216                 break;
7217         case I40E_VIRTCHNL_OP_ADD_VLAN:
7218                 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
7219                 break;
7220         case I40E_VIRTCHNL_OP_DEL_VLAN:
7221                 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
7222                 break;
7223         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
7224                 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
7225                 break;
7226         case I40E_VIRTCHNL_OP_GET_STATS:
7227                 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
7228                 break;
7229
7230         /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
7231         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
7232         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
7233         default:
7234                 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
7235                 break;
7236         }
7237 }
7238
7239 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
7240 static void
7241 ixl_handle_vflr(void *arg, int pending)
7242 {
7243         struct ixl_pf *pf;
7244         struct i40e_hw *hw;
7245         uint16_t global_vf_num;
7246         uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
7247         int i;
7248
7249         pf = arg;
7250         hw = &pf->hw;
7251
7252         IXL_PF_LOCK(pf);
7253         for (i = 0; i < pf->num_vfs; i++) {
7254                 global_vf_num = hw->func_caps.vf_base_id + i;
7255
7256                 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
7257                 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
7258                 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
7259                 if (vflrstat & vflrstat_mask) {
7260                         wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
7261                             vflrstat_mask);
7262
7263                         ixl_reinit_vf(pf, &pf->vfs[i]);
7264                 }
7265         }
7266
7267         icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
7268         icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
7269         wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
7270         ixl_flush(hw);
7271
7272         IXL_PF_UNLOCK(pf);
7273 }
7274
7275 static int
7276 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
7277 {
7278
7279         switch (err) {
7280         case I40E_AQ_RC_EPERM:
7281                 return (EPERM);
7282         case I40E_AQ_RC_ENOENT:
7283                 return (ENOENT);
7284         case I40E_AQ_RC_ESRCH:
7285                 return (ESRCH);
7286         case I40E_AQ_RC_EINTR:
7287                 return (EINTR);
7288         case I40E_AQ_RC_EIO:
7289                 return (EIO);
7290         case I40E_AQ_RC_ENXIO:
7291                 return (ENXIO);
7292         case I40E_AQ_RC_E2BIG:
7293                 return (E2BIG);
7294         case I40E_AQ_RC_EAGAIN:
7295                 return (EAGAIN);
7296         case I40E_AQ_RC_ENOMEM:
7297                 return (ENOMEM);
7298         case I40E_AQ_RC_EACCES:
7299                 return (EACCES);
7300         case I40E_AQ_RC_EFAULT:
7301                 return (EFAULT);
7302         case I40E_AQ_RC_EBUSY:
7303                 return (EBUSY);
7304         case I40E_AQ_RC_EEXIST:
7305                 return (EEXIST);
7306         case I40E_AQ_RC_EINVAL:
7307                 return (EINVAL);
7308         case I40E_AQ_RC_ENOTTY:
7309                 return (ENOTTY);
7310         case I40E_AQ_RC_ENOSPC:
7311                 return (ENOSPC);
7312         case I40E_AQ_RC_ENOSYS:
7313                 return (ENOSYS);
7314         case I40E_AQ_RC_ERANGE:
7315                 return (ERANGE);
7316         case I40E_AQ_RC_EFLUSHED:
7317                 return (EINVAL);        /* No exact equivalent in errno.h */
7318         case I40E_AQ_RC_BAD_ADDR:
7319                 return (EFAULT);
7320         case I40E_AQ_RC_EMODE:
7321                 return (EPERM);
7322         case I40E_AQ_RC_EFBIG:
7323                 return (EFBIG);
7324         default:
7325                 return (EINVAL);
7326         }
7327 }
7328
7329 static int
7330 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
7331 {
7332         struct ixl_pf *pf;
7333         struct i40e_hw *hw;
7334         struct ixl_vsi *pf_vsi;
7335         enum i40e_status_code ret;
7336         int i, error;
7337
7338         pf = device_get_softc(dev);
7339         hw = &pf->hw;
7340         pf_vsi = &pf->vsi;
7341
7342         IXL_PF_LOCK(pf);
7343         pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
7344             M_ZERO);
7345
7346         if (pf->vfs == NULL) {
7347                 error = ENOMEM;
7348                 goto fail;
7349         }
7350
7351         for (i = 0; i < num_vfs; i++)
7352                 sysctl_ctx_init(&pf->vfs[i].ctx);
7353
7354         ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
7355             1, FALSE, &pf->veb_seid, FALSE, NULL);
7356         if (ret != I40E_SUCCESS) {
7357                 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
7358                 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
7359                     error);
7360                 goto fail;
7361         }
7362
7363         // TODO: [Configure MSI-X here]
7364         ixl_enable_adminq(hw);
7365
7366         pf->num_vfs = num_vfs;
7367         IXL_PF_UNLOCK(pf);
7368         return (0);
7369
7370 fail:
7371         free(pf->vfs, M_IXL);
7372         pf->vfs = NULL;
7373         IXL_PF_UNLOCK(pf);
7374         return (error);
7375 }
7376
7377 static void
7378 ixl_iov_uninit(device_t dev)
7379 {
7380         struct ixl_pf *pf;
7381         struct i40e_hw *hw;
7382         struct ixl_vsi *vsi;
7383         struct ifnet *ifp;
7384         struct ixl_vf *vfs;
7385         int i, num_vfs;
7386
7387         pf = device_get_softc(dev);
7388         hw = &pf->hw;
7389         vsi = &pf->vsi;
7390         ifp = vsi->ifp;
7391
7392         IXL_PF_LOCK(pf);
7393         for (i = 0; i < pf->num_vfs; i++) {
7394                 if (pf->vfs[i].vsi.seid != 0)
7395                         i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
7396         }
7397
7398         if (pf->veb_seid != 0) {
7399                 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
7400                 pf->veb_seid = 0;
7401         }
7402
7403         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
7404                 ixl_disable_intr(vsi);
7405
7406         vfs = pf->vfs;
7407         num_vfs = pf->num_vfs;
7408
7409         pf->vfs = NULL;
7410         pf->num_vfs = 0;
7411         IXL_PF_UNLOCK(pf);
7412
7413         /* Do this after the unlock as sysctl_ctx_free might sleep. */
7414         for (i = 0; i < num_vfs; i++)
7415                 sysctl_ctx_free(&vfs[i].ctx);
7416         free(vfs, M_IXL);
7417 }
7418
7419 static int
7420 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
7421 {
7422         char sysctl_name[QUEUE_NAME_LEN];
7423         struct ixl_pf *pf;
7424         struct ixl_vf *vf;
7425         const void *mac;
7426         size_t size;
7427         int error;
7428
7429         pf = device_get_softc(dev);
7430         vf = &pf->vfs[vfnum];
7431
7432         IXL_PF_LOCK(pf);
7433         vf->vf_num = vfnum;
7434
7435         vf->vsi.back = pf;
7436         vf->vf_flags = VF_FLAG_ENABLED;
7437         SLIST_INIT(&vf->vsi.ftl);
7438
7439         error = ixl_vf_setup_vsi(pf, vf);
7440         if (error != 0)
7441                 goto out;
7442
7443         if (nvlist_exists_binary(params, "mac-addr")) {
7444                 mac = nvlist_get_binary(params, "mac-addr", &size);
7445                 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
7446
7447                 if (nvlist_get_bool(params, "allow-set-mac"))
7448                         vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7449         } else
7450                 /*
7451                  * If the administrator has not specified a MAC address then
7452                  * we must allow the VF to choose one.
7453                  */
7454                 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7455
7456         if (nvlist_get_bool(params, "mac-anti-spoof"))
7457                 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
7458
7459         if (nvlist_get_bool(params, "allow-promisc"))
7460                 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
7461
7462         /* TODO: Get VLAN that PF has set for the VF */
7463
7464         vf->vf_flags |= VF_FLAG_VLAN_CAP;
7465
7466         ixl_reset_vf(pf, vf);
7467 out:
7468         IXL_PF_UNLOCK(pf);
7469         if (error == 0) {
7470                 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
7471                 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
7472         }
7473
7474         return (error);
7475 }
7476 #endif /* PCI_IOV */