sys/dev/ixl/if_ixl.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifndef IXL_STANDALONE_BUILD
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "opt_rss.h"
  39 #endif
  40
  41 #include "ixl.h"
  42 #include "ixl_pf.h"
  43
  44 #ifdef RSS
  45 #include <net/rss_config.h>
  46 #endif
  47
  48 /*********************************************************************
  49  *  Driver version
  50  *********************************************************************/
  51 char ixl_driver_version[] = "1.4.27-k";
  52
  53 /*********************************************************************
  54  *  PCI Device ID Table
  55  *
  56  *  Used by probe to select devices to load on
  57  *  Last field stores an index into ixl_strings
  58  *  Last entry must be all 0s
  59  *
  60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  61  *********************************************************************/
  62
  63 static ixl_vendor_info_t ixl_vendor_info_array[] =
  64 {
  65         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
  66         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
  67         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
  68         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
  69         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
  70         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
  71         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
  72         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
  73         /* required last entry */
  74         {0, 0, 0, 0, 0}
  75 };
  76
  77 /*********************************************************************
  78  *  Table of branding strings
  79  *********************************************************************/
  80
  81 static char    *ixl_strings[] = {
  82         "Intel(R) Ethernet Connection XL710 Driver"
  83 };
  84
  85
  86 /*********************************************************************
  87  *  Function prototypes
  88  *********************************************************************/
  89 static int      ixl_probe(device_t);
  90 static int      ixl_attach(device_t);
  91 static int      ixl_detach(device_t);
  92 static int      ixl_shutdown(device_t);
  93 static int      ixl_get_hw_capabilities(struct ixl_pf *);
  94 static void     ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
  95 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
  96 static void     ixl_init(void *);
  97 static void     ixl_init_locked(struct ixl_pf *);
  98 static void     ixl_stop(struct ixl_pf *);
  99 static void     ixl_stop_locked(struct ixl_pf *);
 100 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
 101 static int      ixl_media_change(struct ifnet *);
 102 static void     ixl_update_link_status(struct ixl_pf *);
 103 static int      ixl_allocate_pci_resources(struct ixl_pf *);
 104 static u16      ixl_get_bus_info(struct i40e_hw *, device_t);
 105 static int      ixl_setup_stations(struct ixl_pf *);
 106 static int      ixl_switch_config(struct ixl_pf *);
 107 static int      ixl_initialize_vsi(struct ixl_vsi *);
 108
 109 static int      ixl_setup_adminq_msix(struct ixl_pf *);
 110 static int      ixl_setup_adminq_tq(struct ixl_pf *);
 111 static int      ixl_setup_queue_msix(struct ixl_vsi *);
 112 static int      ixl_setup_queue_tqs(struct ixl_vsi *);
 113 static int      ixl_teardown_adminq_msix(struct ixl_pf *);
 114 static int      ixl_teardown_queue_msix(struct ixl_vsi *);
 115 static void     ixl_configure_intr0_msix(struct ixl_pf *);
 116 static void     ixl_configure_queue_intr_msix(struct ixl_pf *);
 117 static void     ixl_free_queue_tqs(struct ixl_vsi *);
 118 static void     ixl_free_adminq_tq(struct ixl_pf *);
 119
 120 static int      ixl_assign_vsi_legacy(struct ixl_pf *);
 121 static int      ixl_init_msix(struct ixl_pf *);
 122 static void     ixl_configure_itr(struct ixl_pf *);
 123 static void     ixl_configure_legacy(struct ixl_pf *);
 124 static void     ixl_free_pci_resources(struct ixl_pf *);
 125 static void     ixl_local_timer(void *);
 126 static int      ixl_setup_interface(device_t, struct ixl_vsi *);
 127 static void     ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
 128 static void     ixl_config_rss(struct ixl_vsi *);
 129 static void     ixl_set_queue_rx_itr(struct ixl_queue *);
 130 static void     ixl_set_queue_tx_itr(struct ixl_queue *);
 131 static int      ixl_set_advertised_speeds(struct ixl_pf *, int);
 132 static void     ixl_get_initial_advertised_speeds(struct ixl_pf *);
 133
 134 static int      ixl_enable_rings(struct ixl_vsi *);
 135 static int      ixl_disable_rings(struct ixl_vsi *);
 136 static void     ixl_enable_intr(struct ixl_vsi *);
 137 static void     ixl_disable_intr(struct ixl_vsi *);
 138 static void     ixl_disable_rings_intr(struct ixl_vsi *);
 139
 140 static void     ixl_enable_adminq(struct i40e_hw *);
 141 static void     ixl_disable_adminq(struct i40e_hw *);
 142 static void     ixl_enable_queue(struct i40e_hw *, int);
 143 static void     ixl_disable_queue(struct i40e_hw *, int);
 144 static void     ixl_enable_legacy(struct i40e_hw *);
 145 static void     ixl_disable_legacy(struct i40e_hw *);
 146
 147 static void     ixl_set_promisc(struct ixl_vsi *);
 148 static void     ixl_add_multi(struct ixl_vsi *);
 149 static void     ixl_del_multi(struct ixl_vsi *);
 150 static void     ixl_register_vlan(void *, struct ifnet *, u16);
 151 static void     ixl_unregister_vlan(void *, struct ifnet *, u16);
 152 static void     ixl_setup_vlan_filters(struct ixl_vsi *);
 153
 154 static void     ixl_init_filters(struct ixl_vsi *);
 155 static void     ixl_reconfigure_filters(struct ixl_vsi *vsi);
 156 static void     ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
 157 static void     ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
 158 static void     ixl_add_hw_filters(struct ixl_vsi *, int, int);
 159 static void     ixl_del_hw_filters(struct ixl_vsi *, int);
 160 static struct ixl_mac_filter *
 161                 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
 162 static void     ixl_add_mc_filter(struct ixl_vsi *, u8 *);
 163 static void     ixl_free_mac_filters(struct ixl_vsi *vsi);
 164
 165 /* Sysctls*/
 166 static void     ixl_add_device_sysctls(struct ixl_pf *);
 167
 168 static int      ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
 169 static int      ixl_set_advertise(SYSCTL_HANDLER_ARGS);
 170 static int      ixl_current_speed(SYSCTL_HANDLER_ARGS);
 171 static int      ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
 172
 173 #ifdef IXL_DEBUG_SYSCTL
 174 static int      ixl_debug_info(SYSCTL_HANDLER_ARGS);
 175 static void     ixl_print_debug_info(struct ixl_pf *);
 176
 177 static int      ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
 178 static int      ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
 179 static int      ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 180 static int      ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
 181 static int      ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
 182 #endif
 183
 184 /* The MSI/X Interrupt handlers */
 185 static void     ixl_intr(void *);
 186 static void     ixl_msix_que(void *);
 187 static void     ixl_msix_adminq(void *);
 188 static void     ixl_handle_mdd_event(struct ixl_pf *);
 189
 190 /* Deferred interrupt tasklets */
 191 static void     ixl_do_adminq(void *, int);
 192
 193 /* Statistics */
 194 static void     ixl_add_hw_stats(struct ixl_pf *);
 195 static void     ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
 196                     struct sysctl_oid_list *, struct i40e_hw_port_stats *);
 197 static void     ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
 198                     struct sysctl_oid_list *,
 199                     struct i40e_eth_stats *);
 200 static void     ixl_update_stats_counters(struct ixl_pf *);
 201 static void     ixl_update_eth_stats(struct ixl_vsi *);
 202 static void     ixl_update_vsi_stats(struct ixl_vsi *);
 203 static void     ixl_pf_reset_stats(struct ixl_pf *);
 204 static void     ixl_vsi_reset_stats(struct ixl_vsi *);
 205 static void     ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
 206                     u64 *, u64 *);
 207 static void     ixl_stat_update32(struct i40e_hw *, u32, bool,
 208                     u64 *, u64 *);
 209 /* NVM update */
 210 static int      ixl_handle_nvmupd_cmd(struct ixl_pf *, struct ifdrv *);
 211 static void     ixl_handle_empr_reset(struct ixl_pf *);
 212 static int      ixl_rebuild_hw_structs_after_reset(struct ixl_pf *);
 213
 214 /* Debug helper functions */
 215 #ifdef IXL_DEBUG
 216 static void     ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *);
 217 #endif
 218
 219 #ifdef PCI_IOV
 220 static int      ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
 221
 222 static int      ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
 223 static void     ixl_iov_uninit(device_t dev);
 224 static int      ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
 225
 226 static void     ixl_handle_vf_msg(struct ixl_pf *,
 227                     struct i40e_arq_event_info *);
 228 static void     ixl_handle_vflr(void *arg, int pending);
 229
 230 static void     ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 231 static void     ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 232 #endif
 233
 234 /*********************************************************************
 235  *  FreeBSD Device Interface Entry Points
 236  *********************************************************************/
 237
 238 static device_method_t ixl_methods[] = {
 239         /* Device interface */
 240         DEVMETHOD(device_probe, ixl_probe),
 241         DEVMETHOD(device_attach, ixl_attach),
 242         DEVMETHOD(device_detach, ixl_detach),
 243         DEVMETHOD(device_shutdown, ixl_shutdown),
 244 #ifdef PCI_IOV
 245         DEVMETHOD(pci_iov_init, ixl_iov_init),
 246         DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
 247         DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
 248 #endif
 249         {0, 0}
 250 };
 251
 252 static driver_t ixl_driver = {
 253         "ixl", ixl_methods, sizeof(struct ixl_pf),
 254 };
 255
 256 devclass_t ixl_devclass;
 257 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
 258
 259 MODULE_DEPEND(ixl, pci, 1, 1, 1);
 260 MODULE_DEPEND(ixl, ether, 1, 1, 1);
 261 #ifdef DEV_NETMAP
 262 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
 263 #endif /* DEV_NETMAP */
 264
 265 /*
 266 ** Global reset mutex
 267 */
 268 static struct mtx ixl_reset_mtx;
 269
 270 /*
 271 ** TUNEABLE PARAMETERS:
 272 */
 273
 274 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
 275                    "IXL driver parameters");
 276
 277 /*
 278  * MSIX should be the default for best performance,
 279  * but this allows it to be forced off for testing.
 280  */
 281 static int ixl_enable_msix = 1;
 282 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
 283 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
 284     "Enable MSI-X interrupts");
 285
 286 /*
 287 ** Number of descriptors per ring:
 288 **   - TX and RX are the same size
 289 */
 290 static int ixl_ringsz = DEFAULT_RING;
 291 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
 292 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
 293     &ixl_ringsz, 0, "Descriptor Ring Size");
 294
 295 /*
 296 ** This can be set manually, if left as 0 the
 297 ** number of queues will be calculated based
 298 ** on cpus and msix vectors available.
 299 */
 300 int ixl_max_queues = 0;
 301 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
 302 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
 303     &ixl_max_queues, 0, "Number of Queues");
 304
 305 /*
 306 ** Controls for Interrupt Throttling
 307 **      - true/false for dynamic adjustment
 308 **      - default values for static ITR
 309 */
 310 int ixl_dynamic_rx_itr = 1;
 311 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
 312 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
 313     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
 314
 315 int ixl_dynamic_tx_itr = 1;
 316 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
 317 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
 318     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
 319
 320 int ixl_rx_itr = IXL_ITR_8K;
 321 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
 322 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
 323     &ixl_rx_itr, 0, "RX Interrupt Rate");
 324
 325 int ixl_tx_itr = IXL_ITR_4K;
 326 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
 327 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
 328     &ixl_tx_itr, 0, "TX Interrupt Rate");
 329
 330 #ifdef IXL_FDIR
 331 static int ixl_enable_fdir = 1;
 332 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
 333 /* Rate at which we sample */
 334 int ixl_atr_rate = 20;
 335 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
 336 #endif
 337
 338 #ifdef DEV_NETMAP
 339 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
 340 #include <dev/netmap/if_ixl_netmap.h>
 341 #endif /* DEV_NETMAP */
 342
 343 static char *ixl_fc_string[6] = {
 344         "None",
 345         "Rx",
 346         "Tx",
 347         "Full",
 348         "Priority",
 349         "Default"
 350 };
 351
 352 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
 353
 354 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
 355     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 356
 357 /*********************************************************************
 358  *  Device identification routine
 359  *
 360  *  ixl_probe determines if the driver should be loaded on
 361  *  the hardware based on PCI vendor/device id of the device.
 362  *
 363  *  return BUS_PROBE_DEFAULT on success, positive on failure
 364  *********************************************************************/
 365
 366 static int
 367 ixl_probe(device_t dev)
 368 {
 369         ixl_vendor_info_t *ent;
 370
 371         u16     pci_vendor_id, pci_device_id;
 372         u16     pci_subvendor_id, pci_subdevice_id;
 373         char    device_name[256];
 374         static bool lock_init = FALSE;
 375
 376 #if 0
 377         INIT_DEBUGOUT("ixl_probe: begin");
 378 #endif
 379         pci_vendor_id = pci_get_vendor(dev);
 380         if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
 381                 return (ENXIO);
 382
 383         pci_device_id = pci_get_device(dev);
 384         pci_subvendor_id = pci_get_subvendor(dev);
 385         pci_subdevice_id = pci_get_subdevice(dev);
 386
 387         ent = ixl_vendor_info_array;
 388         while (ent->vendor_id != 0) {
 389                 if ((pci_vendor_id == ent->vendor_id) &&
 390                     (pci_device_id == ent->device_id) &&
 391
 392                     ((pci_subvendor_id == ent->subvendor_id) ||
 393                      (ent->subvendor_id == 0)) &&
 394
 395                     ((pci_subdevice_id == ent->subdevice_id) ||
 396                      (ent->subdevice_id == 0))) {
 397                         sprintf(device_name, "%s, Version - %s",
 398                                 ixl_strings[ent->index],
 399                                 ixl_driver_version);
 400                         device_set_desc_copy(dev, device_name);
 401                         /* One shot mutex init */
 402                         if (lock_init == FALSE) {
 403                                 lock_init = TRUE;
 404                                 mtx_init(&ixl_reset_mtx,
 405                                     "ixl_reset",
 406                                     "IXL RESET Lock", MTX_DEF);
 407                         }
 408                         return (BUS_PROBE_DEFAULT);
 409                 }
 410                 ent++;
 411         }
 412         return (ENXIO);
 413 }
 414
 415 /*********************************************************************
 416  *  Device initialization routine
 417  *
 418  *  The attach entry point is called when the driver is being loaded.
 419  *  This routine identifies the type of hardware, allocates all resources
 420  *  and initializes the hardware.
 421  *
 422  *  return 0 on success, positive on failure
 423  *********************************************************************/
 424
 425 static int
 426 ixl_attach(device_t dev)
 427 {
 428         struct ixl_pf   *pf;
 429         struct i40e_hw  *hw;
 430         struct ixl_vsi  *vsi;
 431         u16             bus;
 432         int             error = 0;
 433 #ifdef PCI_IOV
 434         nvlist_t        *pf_schema, *vf_schema;
 435         int             iov_error;
 436 #endif
 437
 438         INIT_DEBUGOUT("ixl_attach: begin");
 439
 440         /* Allocate, clear, and link in our primary soft structure */
 441         pf = device_get_softc(dev);
 442         pf->dev = pf->osdep.dev = dev;
 443         hw = &pf->hw;
 444
 445         /*
 446         ** Note this assumes we have a single embedded VSI,
 447         ** this could be enhanced later to allocate multiple
 448         */
 449         vsi = &pf->vsi;
 450         vsi->dev = pf->dev;
 451
 452         /* Core Lock Init*/
 453         IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
 454
 455         /* Set up the timer callout */
 456         callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
 457
 458         /* Save off the PCI information */
 459         hw->vendor_id = pci_get_vendor(dev);
 460         hw->device_id = pci_get_device(dev);
 461         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
 462         hw->subsystem_vendor_id =
 463             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 464         hw->subsystem_device_id =
 465             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 466
 467         hw->bus.device = pci_get_slot(dev);
 468         hw->bus.func = pci_get_function(dev);
 469
 470         pf->vc_debug_lvl = 1;
 471
 472         /* Do PCI setup - map BAR0, etc */
 473         if (ixl_allocate_pci_resources(pf)) {
 474                 device_printf(dev, "Allocation of PCI resources failed\n");
 475                 error = ENXIO;
 476                 goto err_out;
 477         }
 478
 479         /* Establish a clean starting point */
 480         i40e_clear_hw(hw);
 481         error = i40e_pf_reset(hw);
 482         if (error) {
 483                 device_printf(dev, "PF reset failure %d\n", error);
 484                 error = EIO;
 485                 goto err_out;
 486         }
 487
 488         /* Set admin queue parameters */
 489         hw->aq.num_arq_entries = IXL_AQ_LEN;
 490         hw->aq.num_asq_entries = IXL_AQ_LEN;
 491         hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
 492         hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
 493
 494         /* Initialize mac filter list for VSI */
 495         SLIST_INIT(&vsi->ftl);
 496
 497         /* Initialize the shared code */
 498         error = i40e_init_shared_code(hw);
 499         if (error) {
 500                 device_printf(dev, "Unable to initialize shared code, error %d\n",
 501                     error);
 502                 error = EIO;
 503                 goto err_out;
 504         }
 505
 506         /* Set up the admin queue */
 507         error = i40e_init_adminq(hw);
 508         if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
 509                 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
 510                     error);
 511                 error = EIO;
 512                 goto err_out;
 513         }
 514         ixl_print_nvm_version(pf);
 515
 516         if (error == I40E_ERR_FIRMWARE_API_VERSION) {
 517                 device_printf(dev, "The driver for the device stopped "
 518                     "because the NVM image is newer than expected.\n"
 519                     "You must install the most recent version of "
 520                     "the network driver.\n");
 521                 error = EIO;
 522                 goto err_out;
 523         }
 524
 525         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
 526             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
 527                 device_printf(dev, "The driver for the device detected "
 528                     "a newer version of the NVM image than expected.\n"
 529                     "Please install the most recent version of the network driver.\n");
 530         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
 531             hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
 532                 device_printf(dev, "The driver for the device detected "
 533                     "an older version of the NVM image than expected.\n"
 534                     "Please update the NVM image.\n");
 535
 536         /* Clear PXE mode */
 537         i40e_clear_pxe_mode(hw);
 538
 539         /* Get capabilities from the device */
 540         error = ixl_get_hw_capabilities(pf);
 541         if (error) {
 542                 device_printf(dev, "HW capabilities failure!\n");
 543                 goto err_get_cap;
 544         }
 545
 546         /* Set up host memory cache */
 547         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 548             hw->func_caps.num_rx_qp, 0, 0);
 549         if (error) {
 550                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
 551                 goto err_get_cap;
 552         }
 553
 554         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 555         if (error) {
 556                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
 557                 goto err_mac_hmc;
 558         }
 559
 560         /* Disable LLDP from the firmware for certain NVM versions */
 561         if (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
 562             (pf->hw.aq.fw_maj_ver < 4))
 563                 i40e_aq_stop_lldp(hw, TRUE, NULL);
 564
 565         i40e_get_mac_addr(hw, hw->mac.addr);
 566         error = i40e_validate_mac_addr(hw->mac.addr);
 567         if (error) {
 568                 device_printf(dev, "validate_mac_addr failed: %d\n", error);
 569                 goto err_mac_hmc;
 570         }
 571         bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
 572         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 573
 574         /* Set up VSI and queues */
 575         if (ixl_setup_stations(pf) != 0) {
 576                 device_printf(dev, "setup stations failed!\n");
 577                 error = ENOMEM;
 578                 goto err_mac_hmc;
 579         }
 580
 581         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 582             (hw->aq.fw_maj_ver < 4)) {
 583                 i40e_msec_delay(75);
 584                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 585                 if (error) {
 586                         device_printf(dev, "link restart failed, aq_err=%d\n",
 587                             pf->hw.aq.asq_last_status);
 588                         goto err_late;
 589                 }
 590         }
 591
 592         /* Determine link state */
 593         hw->phy.get_link_info = TRUE;
 594         i40e_get_link_status(hw, &pf->link_up);
 595
 596         /* Setup OS network interface / ifnet */
 597         if (ixl_setup_interface(dev, vsi) != 0) {
 598                 device_printf(dev, "interface setup failed!\n");
 599                 error = EIO;
 600                 goto err_late;
 601         }
 602
 603         error = ixl_switch_config(pf);
 604         if (error) {
 605                 device_printf(dev, "Initial ixl_switch_config() failed: %d\n",
 606                      error);
 607                 goto err_late;
 608         }
 609
 610         /* Limit PHY interrupts to link, autoneg, and modules failure */
 611         error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
 612             NULL);
 613         if (error) {
 614                 device_printf(dev, "i40e_aq_set_phy_mask() failed: err %d,"
 615                     " aq_err %d\n", error, hw->aq.asq_last_status);
 616                 goto err_late;
 617         }
 618
 619         /* Get the bus configuration and set the shared code's config */
 620         bus = ixl_get_bus_info(hw, dev);
 621         i40e_set_pci_config_data(hw, bus);
 622
 623         /*
 624          * In MSI-X mode, initialize the Admin Queue interrupt,
 625          * so userland tools can communicate with the adapter regardless of
 626          * the ifnet interface's status.
 627          */
 628         if (pf->msix > 1) {
 629                 error = ixl_setup_adminq_msix(pf);
 630                 if (error) {
 631                         device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
 632                             error);
 633                         goto err_late;
 634                 }
 635                 error = ixl_setup_adminq_tq(pf);
 636                 if (error) {
 637                         device_printf(dev, "ixl_setup_adminq_tq error: %d\n",
 638                             error);
 639                         goto err_late;
 640                 }
 641                 ixl_configure_intr0_msix(pf);
 642                 ixl_enable_adminq(hw);
 643         }
 644
 645         /* Initialize statistics & add sysctls */
 646         ixl_add_device_sysctls(pf);
 647
 648         ixl_pf_reset_stats(pf);
 649         ixl_update_stats_counters(pf);
 650         ixl_add_hw_stats(pf);
 651
 652         /* Register for VLAN events */
 653         vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 654             ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 655         vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 656             ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 657
 658 #ifdef PCI_IOV
 659         /* SR-IOV is only supported when MSI-X is in use. */
 660         if (pf->msix > 1) {
 661                 pf_schema = pci_iov_schema_alloc_node();
 662                 vf_schema = pci_iov_schema_alloc_node();
 663                 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
 664                 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
 665                     IOV_SCHEMA_HASDEFAULT, TRUE);
 666                 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
 667                     IOV_SCHEMA_HASDEFAULT, FALSE);
 668                 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
 669                     IOV_SCHEMA_HASDEFAULT, FALSE);
 670
 671                 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
 672                 if (iov_error != 0) {
 673                         device_printf(dev,
 674                             "Failed to initialize SR-IOV (error=%d)\n",
 675                             iov_error);
 676                 } else
 677                         device_printf(dev, "SR-IOV ready\n");
 678         }
 679 #endif
 680
 681 #ifdef DEV_NETMAP
 682         ixl_netmap_attach(vsi);
 683 #endif /* DEV_NETMAP */
 684         INIT_DEBUGOUT("ixl_attach: end");
 685         return (0);
 686
 687 err_late:
 688         if (vsi->ifp != NULL)
 689                 if_free(vsi->ifp);
 690 err_mac_hmc:
 691         i40e_shutdown_lan_hmc(hw);
 692 err_get_cap:
 693         i40e_shutdown_adminq(hw);
 694 err_out:
 695         ixl_free_pci_resources(pf);
 696         ixl_free_vsi(vsi);
 697         IXL_PF_LOCK_DESTROY(pf);
 698         return (error);
 699 }
 700
 701 /*********************************************************************
 702  *  Device removal routine
 703  *
 704  *  The detach entry point is called when the driver is being removed.
 705  *  This routine stops the adapter and deallocates all the resources
 706  *  that were allocated for driver operation.
 707  *
 708  *  return 0 on success, positive on failure
 709  *********************************************************************/
 710
 711 static int
 712 ixl_detach(device_t dev)
 713 {
 714         struct ixl_pf           *pf = device_get_softc(dev);
 715         struct i40e_hw          *hw = &pf->hw;
 716         struct ixl_vsi          *vsi = &pf->vsi;
 717         enum i40e_status_code   status;
 718 #ifdef PCI_IOV
 719         int                     error;
 720 #endif
 721
 722         INIT_DEBUGOUT("ixl_detach: begin");
 723
 724         /* Make sure VLANS are not using driver */
 725         if (vsi->ifp->if_vlantrunk != NULL) {
 726                 device_printf(dev, "Vlan in use, detach first\n");
 727                 return (EBUSY);
 728         }
 729
 730 #ifdef PCI_IOV
 731         error = pci_iov_detach(dev);
 732         if (error != 0) {
 733                 device_printf(dev, "SR-IOV in use; detach first.\n");
 734                 return (error);
 735         }
 736 #endif
 737
 738         ether_ifdetach(vsi->ifp);
 739         if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)
 740                 ixl_stop(pf);
 741
 742         ixl_free_queue_tqs(vsi);
 743
 744         /* Shutdown LAN HMC */
 745         status = i40e_shutdown_lan_hmc(hw);
 746         if (status)
 747                 device_printf(dev,
 748                     "Shutdown LAN HMC failed with code %d\n", status);
 749
 750         /* Shutdown admin queue */
 751         ixl_disable_adminq(hw);
 752         ixl_free_adminq_tq(pf);
 753         ixl_teardown_adminq_msix(pf);
 754         status = i40e_shutdown_adminq(hw);
 755         if (status)
 756                 device_printf(dev,
 757                     "Shutdown Admin queue failed with code %d\n", status);
 758
 759         /* Unregister VLAN events */
 760         if (vsi->vlan_attach != NULL)
 761                 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
 762         if (vsi->vlan_detach != NULL)
 763                 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
 764
 765         callout_drain(&pf->timer);
 766 #ifdef DEV_NETMAP
 767         netmap_detach(vsi->ifp);
 768 #endif /* DEV_NETMAP */
 769         ixl_free_pci_resources(pf);
 770         bus_generic_detach(dev);
 771         if_free(vsi->ifp);
 772         ixl_free_vsi(vsi);
 773         IXL_PF_LOCK_DESTROY(pf);
 774         return (0);
 775 }
 776
 777 /*********************************************************************
 778  *
 779  *  Shutdown entry point
 780  *
 781  **********************************************************************/
 782
 783 static int
 784 ixl_shutdown(device_t dev)
 785 {
 786         struct ixl_pf *pf = device_get_softc(dev);
 787         ixl_stop(pf);
 788         return (0);
 789 }
 790
 791
 792 /*********************************************************************
 793  *
 794  *  Get the hardware capabilities
 795  *
 796  **********************************************************************/
 797
 798 static int
 799 ixl_get_hw_capabilities(struct ixl_pf *pf)
 800 {
 801         struct i40e_aqc_list_capabilities_element_resp *buf;
 802         struct i40e_hw  *hw = &pf->hw;
 803         device_t        dev = pf->dev;
 804         int             error, len;
 805         u16             needed;
 806         bool            again = TRUE;
 807
 808         len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
 809 retry:
 810         if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
 811             malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 812                 device_printf(dev, "Unable to allocate cap memory\n");
 813                 return (ENOMEM);
 814         }
 815
 816         /* This populates the hw struct */
 817         error = i40e_aq_discover_capabilities(hw, buf, len,
 818             &needed, i40e_aqc_opc_list_func_capabilities, NULL);
 819         free(buf, M_DEVBUF);
 820         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
 821             (again == TRUE)) {
 822                 /* retry once with a larger buffer */
 823                 again = FALSE;
 824                 len = needed;
 825                 goto retry;
 826         } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
 827                 device_printf(dev, "capability discovery failed: %d\n",
 828                     pf->hw.aq.asq_last_status);
 829                 return (ENODEV);
 830         }
 831
 832         /* Capture this PF's starting queue pair */
 833         pf->qbase = hw->func_caps.base_queue;
 834
 835 #ifdef IXL_DEBUG
 836         device_printf(dev, "pf_id=%d, num_vfs=%d, msix_pf=%d, "
 837             "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
 838             hw->pf_id, hw->func_caps.num_vfs,
 839             hw->func_caps.num_msix_vectors,
 840             hw->func_caps.num_msix_vectors_vf,
 841             hw->func_caps.fd_filters_guaranteed,
 842             hw->func_caps.fd_filters_best_effort,
 843             hw->func_caps.num_tx_qp,
 844             hw->func_caps.num_rx_qp,
 845             hw->func_caps.base_queue);
 846 #endif
 847         return (error);
 848 }
 849
 850 static void
 851 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
 852 {
 853         device_t        dev = vsi->dev;
 854
 855         /* Enable/disable TXCSUM/TSO4 */
 856         if (!(ifp->if_capenable & IFCAP_TXCSUM)
 857             && !(ifp->if_capenable & IFCAP_TSO4)) {
 858                 if (mask & IFCAP_TXCSUM) {
 859                         ifp->if_capenable |= IFCAP_TXCSUM;
 860                         /* enable TXCSUM, restore TSO if previously enabled */
 861                         if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
 862                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 863                                 ifp->if_capenable |= IFCAP_TSO4;
 864                         }
 865                 }
 866                 else if (mask & IFCAP_TSO4) {
 867                         ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
 868                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 869                         device_printf(dev,
 870                             "TSO4 requires txcsum, enabling both...\n");
 871                 }
 872         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 873             && !(ifp->if_capenable & IFCAP_TSO4)) {
 874                 if (mask & IFCAP_TXCSUM)
 875                         ifp->if_capenable &= ~IFCAP_TXCSUM;
 876                 else if (mask & IFCAP_TSO4)
 877                         ifp->if_capenable |= IFCAP_TSO4;
 878         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 879             && (ifp->if_capenable & IFCAP_TSO4)) {
 880                 if (mask & IFCAP_TXCSUM) {
 881                         vsi->flags |= IXL_FLAGS_KEEP_TSO4;
 882                         ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
 883                         device_printf(dev,
 884                             "TSO4 requires txcsum, disabling both...\n");
 885                 } else if (mask & IFCAP_TSO4)
 886                         ifp->if_capenable &= ~IFCAP_TSO4;
 887         }
 888
 889         /* Enable/disable TXCSUM_IPV6/TSO6 */
 890         if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 891             && !(ifp->if_capenable & IFCAP_TSO6)) {
 892                 if (mask & IFCAP_TXCSUM_IPV6) {
 893                         ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
 894                         if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
 895                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 896                                 ifp->if_capenable |= IFCAP_TSO6;
 897                         }
 898                 } else if (mask & IFCAP_TSO6) {
 899                         ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 900                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 901                         device_printf(dev,
 902                             "TSO6 requires txcsum6, enabling both...\n");
 903                 }
 904         } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 905             && !(ifp->if_capenable & IFCAP_TSO6)) {
 906                 if (mask & IFCAP_TXCSUM_IPV6)
 907                         ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
 908                 else if (mask & IFCAP_TSO6)
 909                         ifp->if_capenable |= IFCAP_TSO6;
 910         } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 911             && (ifp->if_capenable & IFCAP_TSO6)) {
 912                 if (mask & IFCAP_TXCSUM_IPV6) {
 913                         vsi->flags |= IXL_FLAGS_KEEP_TSO6;
 914                         ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 915                         device_printf(dev,
 916                             "TSO6 requires txcsum6, disabling both...\n");
 917                 } else if (mask & IFCAP_TSO6)
 918                         ifp->if_capenable &= ~IFCAP_TSO6;
 919         }
 920 }
 921
 922 /*********************************************************************
 923  *  Ioctl entry point
 924  *
 925  *  ixl_ioctl is called when the user wants to configure the
 926  *  interface.
 927  *
 928  *  return 0 on success, positive on failure
 929  **********************************************************************/
 930
 931 static int
 932 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 933 {
 934         struct ixl_vsi  *vsi = ifp->if_softc;
 935         struct ixl_pf   *pf = vsi->back;
 936         struct ifreq    *ifr = (struct ifreq *)data;
 937         struct ifdrv    *ifd = (struct ifdrv *)data;
 938 #if defined(INET) || defined(INET6)
 939         struct ifaddr *ifa = (struct ifaddr *)data;
 940         bool            avoid_reset = FALSE;
 941 #endif
 942         int             error = 0;
 943
 944         switch (command) {
 945
 946         case SIOCSIFADDR:
 947 #ifdef INET
 948                 if (ifa->ifa_addr->sa_family == AF_INET)
 949                         avoid_reset = TRUE;
 950 #endif
 951 #ifdef INET6
 952                 if (ifa->ifa_addr->sa_family == AF_INET6)
 953                         avoid_reset = TRUE;
 954 #endif
 955 #if defined(INET) || defined(INET6)
 956                 /*
 957                 ** Calling init results in link renegotiation,
 958                 ** so we avoid doing it when possible.
 959                 */
 960                 if (avoid_reset) {
 961                         ifp->if_flags |= IFF_UP;
 962                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 963                                 ixl_init(pf);
 964 #ifdef INET
 965                         if (!(ifp->if_flags & IFF_NOARP))
 966                                 arp_ifinit(ifp, ifa);
 967 #endif
 968                 } else
 969                         error = ether_ioctl(ifp, command, data);
 970                 break;
 971 #endif
 972         case SIOCSIFMTU:
 973                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 974                 if (ifr->ifr_mtu > IXL_MAX_FRAME -
 975                    ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
 976                         error = EINVAL;
 977                 } else {
 978                         IXL_PF_LOCK(pf);
 979                         ifp->if_mtu = ifr->ifr_mtu;
 980                         vsi->max_frame_size =
 981                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
 982                             + ETHER_VLAN_ENCAP_LEN;
 983                         ixl_init_locked(pf);
 984                         IXL_PF_UNLOCK(pf);
 985                 }
 986                 break;
 987         case SIOCSIFFLAGS:
 988                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 989                 IXL_PF_LOCK(pf);
 990                 if (ifp->if_flags & IFF_UP) {
 991                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 992                                 if ((ifp->if_flags ^ pf->if_flags) &
 993                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 994                                         ixl_set_promisc(vsi);
 995                                 }
 996                         } else {
 997                                 IXL_PF_UNLOCK(pf);
 998                                 ixl_init(pf);
 999                                 IXL_PF_LOCK(pf);
1000                         }
1001                 } else {
1002                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1003                                 IXL_PF_UNLOCK(pf);
1004                                 ixl_stop(pf);
1005                                 IXL_PF_LOCK(pf);
1006                         }
1007                 }
1008                 pf->if_flags = ifp->if_flags;
1009                 IXL_PF_UNLOCK(pf);
1010                 break;
1011         case SIOCSDRVSPEC:
1012         case SIOCGDRVSPEC:
1013                 IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific "
1014                     "Info)\n");
1015
1016                 /* NVM update command */
1017                 if (ifd->ifd_cmd == I40E_NVM_ACCESS)
1018                         error = ixl_handle_nvmupd_cmd(pf, ifd);
1019                 else
1020                         error = EINVAL;
1021                 break;
1022         case SIOCADDMULTI:
1023                 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1024                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1025                         IXL_PF_LOCK(pf);
1026                         ixl_disable_intr(vsi);
1027                         ixl_add_multi(vsi);
1028                         ixl_enable_intr(vsi);
1029                         IXL_PF_UNLOCK(pf);
1030                 }
1031                 break;
1032         case SIOCDELMULTI:
1033                 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1034                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1035                         IXL_PF_LOCK(pf);
1036                         ixl_disable_intr(vsi);
1037                         ixl_del_multi(vsi);
1038                         ixl_enable_intr(vsi);
1039                         IXL_PF_UNLOCK(pf);
1040                 }
1041                 break;
1042         case SIOCSIFMEDIA:
1043         case SIOCGIFMEDIA:
1044 #ifdef IFM_ETH_XTYPE
1045         case SIOCGIFXMEDIA:
1046 #endif
1047                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1048                 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1049                 break;
1050         case SIOCSIFCAP:
1051         {
1052                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1053                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1054
1055                 ixl_cap_txcsum_tso(vsi, ifp, mask);
1056
1057                 if (mask & IFCAP_RXCSUM)
1058                         ifp->if_capenable ^= IFCAP_RXCSUM;
1059                 if (mask & IFCAP_RXCSUM_IPV6)
1060                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1061                 if (mask & IFCAP_LRO)
1062                         ifp->if_capenable ^= IFCAP_LRO;
1063                 if (mask & IFCAP_VLAN_HWTAGGING)
1064                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1065                 if (mask & IFCAP_VLAN_HWFILTER)
1066                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1067                 if (mask & IFCAP_VLAN_HWTSO)
1068                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1069                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1070                         IXL_PF_LOCK(pf);
1071                         ixl_init_locked(pf);
1072                         IXL_PF_UNLOCK(pf);
1073                 }
1074                 VLAN_CAPABILITIES(ifp);
1075
1076                 break;
1077         }
1078
1079         default:
1080                 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1081                 error = ether_ioctl(ifp, command, data);
1082                 break;
1083         }
1084
1085         return (error);
1086 }
1087
1088
1089 /*********************************************************************
1090  *  Init entry point
1091  *
1092  *  This routine is used in two ways. It is used by the stack as
1093  *  init entry point in network interface structure. It is also used
1094  *  by the driver as a hw/sw initialization routine to get to a
1095  *  consistent state.
1096  *
1097  *  return 0 on success, positive on failure
1098  **********************************************************************/
1099
1100 static void
1101 ixl_init_locked(struct ixl_pf *pf)
1102 {
1103         struct i40e_hw  *hw = &pf->hw;
1104         struct ixl_vsi  *vsi = &pf->vsi;
1105         struct ifnet    *ifp = vsi->ifp;
1106         device_t        dev = pf->dev;
1107         struct i40e_filter_control_settings     filter;
1108         u8              tmpaddr[ETHER_ADDR_LEN];
1109         int             ret;
1110
1111         mtx_assert(&pf->pf_mtx, MA_OWNED);
1112         INIT_DEBUGOUT("ixl_init_locked: begin");
1113
1114         ixl_stop_locked(pf);
1115
1116         /* Get the latest mac address... User might use a LAA */
1117         bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1118               I40E_ETH_LENGTH_OF_ADDRESS);
1119         if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1120             (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
1121                 ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1122                 bcopy(tmpaddr, hw->mac.addr,
1123                     I40E_ETH_LENGTH_OF_ADDRESS);
1124                 ret = i40e_aq_mac_address_write(hw,
1125                     I40E_AQC_WRITE_TYPE_LAA_ONLY,
1126                     hw->mac.addr, NULL);
1127                 if (ret) {
1128                         device_printf(dev, "LLA address"
1129                          "change failed!!\n");
1130                         return;
1131                 }
1132         }
1133
1134         ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1135
1136         /* Set the various hardware offload abilities */
1137         ifp->if_hwassist = 0;
1138         if (ifp->if_capenable & IFCAP_TSO)
1139                 ifp->if_hwassist |= CSUM_TSO;
1140         if (ifp->if_capenable & IFCAP_TXCSUM)
1141                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1142         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1143                 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1144
1145         /* Set up the device filtering */
1146         bzero(&filter, sizeof(filter));
1147         filter.enable_ethtype = TRUE;
1148         filter.enable_macvlan = TRUE;
1149 #ifdef IXL_FDIR
1150         filter.enable_fdir = TRUE;
1151 #endif
1152         filter.hash_lut_size = I40E_HASH_LUT_SIZE_512;
1153         if (i40e_set_filter_control(hw, &filter))
1154                 device_printf(dev, "i40e_set_filter_control() failed\n");
1155
1156         /* Set up RSS */
1157         ixl_config_rss(vsi);
1158
1159         /* Prepare the VSI: rings, hmc contexts, etc... */
1160         if (ixl_initialize_vsi(vsi)) {
1161                 device_printf(dev, "initialize vsi failed!!\n");
1162                 return;
1163         }
1164
1165         /* Add protocol filters to list */
1166         ixl_init_filters(vsi);
1167
1168         /* Setup vlan's if needed */
1169         ixl_setup_vlan_filters(vsi);
1170
1171         /* Set up MSI/X routing and the ITR settings */
1172         if (ixl_enable_msix) {
1173                 ixl_configure_queue_intr_msix(pf);
1174                 ixl_configure_itr(pf);
1175         } else
1176                 ixl_configure_legacy(pf);
1177
1178         ixl_enable_rings(vsi);
1179
1180         i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1181
1182         ixl_reconfigure_filters(vsi);
1183
1184         /* And now turn on interrupts */
1185         ixl_enable_intr(vsi);
1186
1187         /* Get link info */
1188         hw->phy.get_link_info = TRUE;
1189         i40e_get_link_status(hw, &pf->link_up);
1190         ixl_update_link_status(pf);
1191
1192         /* Set initial advertised speed sysctl value */
1193         ixl_get_initial_advertised_speeds(pf);
1194
1195         /* Start the local timer */
1196         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1197
1198         /* Now inform the stack we're ready */
1199         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1200
1201         return;
1202 }
1203
1204 /* For the set_advertise sysctl */
1205 static void
1206 ixl_get_initial_advertised_speeds(struct ixl_pf *pf)
1207 {
1208         struct i40e_hw *hw = &pf->hw;
1209         device_t dev = pf->dev;
1210         enum i40e_status_code status;
1211         struct i40e_aq_get_phy_abilities_resp abilities;
1212
1213         /* Set initial sysctl values */
1214         status = i40e_aq_get_phy_capabilities(hw, FALSE, false, &abilities,
1215                                               NULL);
1216         if (status) {
1217                 /* Non-fatal error */
1218                 device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n",
1219                      __func__, status);
1220                 return;
1221         }
1222
1223         if (abilities.link_speed & I40E_LINK_SPEED_40GB)
1224                 pf->advertised_speed |= 0x10;
1225         if (abilities.link_speed & I40E_LINK_SPEED_20GB)
1226                 pf->advertised_speed |= 0x8;
1227         if (abilities.link_speed & I40E_LINK_SPEED_10GB)
1228                 pf->advertised_speed |= 0x4;
1229         if (abilities.link_speed & I40E_LINK_SPEED_1GB)
1230                 pf->advertised_speed |= 0x2;
1231         if (abilities.link_speed & I40E_LINK_SPEED_100MB)
1232                 pf->advertised_speed |= 0x1;
1233 }
1234
1235 static int
1236 ixl_teardown_hw_structs(struct ixl_pf *pf)
1237 {
1238         enum i40e_status_code status = 0;
1239         struct i40e_hw *hw = &pf->hw;
1240         device_t dev = pf->dev;
1241
1242         /* Shutdown LAN HMC */
1243         if (hw->hmc.hmc_obj) {
1244                 status = i40e_shutdown_lan_hmc(hw);
1245                 if (status) {
1246                         device_printf(dev,
1247                             "init: LAN HMC shutdown failure; status %d\n", status);
1248                         goto err_out;
1249                 }
1250         }
1251
1252         // XXX: This gets called when we know the adminq is inactive;
1253         // so we already know it's setup when we get here.
1254
1255         /* Shutdown admin queue */
1256         status = i40e_shutdown_adminq(hw);
1257         if (status)
1258                 device_printf(dev,
1259                     "init: Admin Queue shutdown failure; status %d\n", status);
1260
1261 err_out:
1262         return (status);
1263 }
1264
1265 static int
1266 ixl_reset(struct ixl_pf *pf)
1267 {
1268         struct i40e_hw *hw = &pf->hw;
1269         device_t dev = pf->dev;
1270         u8 set_fc_err_mask;
1271         int error = 0;
1272
1273         // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary
1274         i40e_clear_hw(hw);
1275         error = i40e_pf_reset(hw);
1276         if (error) {
1277                 device_printf(dev, "init: PF reset failure");
1278                 error = EIO;
1279                 goto err_out;
1280         }
1281
1282         error = i40e_init_adminq(hw);
1283         if (error) {
1284                 device_printf(dev, "init: Admin queue init failure;"
1285                     " status code %d", error);
1286                 error = EIO;
1287                 goto err_out;
1288         }
1289
1290         i40e_clear_pxe_mode(hw);
1291
1292         error = ixl_get_hw_capabilities(pf);
1293         if (error) {
1294                 device_printf(dev, "init: Error retrieving HW capabilities;"
1295                     " status code %d\n", error);
1296                 goto err_out;
1297         }
1298
1299         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1300             hw->func_caps.num_rx_qp, 0, 0);
1301         if (error) {
1302                 device_printf(dev, "init: LAN HMC init failed; status code %d\n",
1303                     error);
1304                 error = EIO;
1305                 goto err_out;
1306         }
1307
1308         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1309         if (error) {
1310                 device_printf(dev, "init: LAN HMC config failed; status code %d\n",
1311                     error);
1312                 error = EIO;
1313                 goto err_out;
1314         }
1315
1316         // XXX: possible fix for panic, but our failure recovery is still broken
1317         error = ixl_switch_config(pf);
1318         if (error) {
1319                 device_printf(dev, "init: ixl_switch_config() failed: %d\n",
1320                      error);
1321                 goto err_out;
1322         }
1323
1324         error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
1325             NULL);
1326         if (error) {
1327                 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
1328                     " aq_err %d\n", error, hw->aq.asq_last_status);
1329                 error = EIO;
1330                 goto err_out;
1331         }
1332
1333         error = i40e_set_fc(hw, &set_fc_err_mask, true);
1334         if (error) {
1335                 device_printf(dev, "init: setting link flow control failed; retcode %d,"
1336                     " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
1337                 goto err_out;
1338         }
1339
1340         // XXX: (Rebuild VSIs?)
1341
1342         /* Firmware delay workaround */
1343         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
1344             (hw->aq.fw_maj_ver < 4)) {
1345                 i40e_msec_delay(75);
1346                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
1347                 if (error) {
1348                         device_printf(dev, "init: link restart failed, aq_err %d\n",
1349                             hw->aq.asq_last_status);
1350                         goto err_out;
1351                 }
1352         }
1353
1354
1355 err_out:
1356         return (error);
1357 }
1358
1359 static void
1360 ixl_init(void *arg)
1361 {
1362         struct ixl_pf *pf = arg;
1363         struct ixl_vsi *vsi = &pf->vsi;
1364         device_t dev = pf->dev;
1365         int error = 0;
1366
1367         /*
1368          * If the aq is dead here, it probably means something outside of the driver
1369          * did something to the adapter, like a PF reset.
1370          * So rebuild the driver's state here if that occurs.
1371          */
1372         if (!i40e_check_asq_alive(&pf->hw)) {
1373                 device_printf(dev, "Admin Queue is down; resetting...\n");
1374                 IXL_PF_LOCK(pf);
1375                 ixl_teardown_hw_structs(pf);
1376                 ixl_reset(pf);
1377                 IXL_PF_UNLOCK(pf);
1378         }
1379
1380         /*
1381          * Set up LAN queue interrupts here.
1382          * Kernel interrupt setup functions cannot be called while holding a lock,
1383          * so this is done outside of init_locked().
1384          */
1385         if (pf->msix > 1) {
1386                 /* Teardown existing interrupts, if they exist */
1387                 ixl_teardown_queue_msix(vsi);
1388                 ixl_free_queue_tqs(vsi);
1389                 /* Then set them up again */
1390                 error = ixl_setup_queue_msix(vsi);
1391                 if (error)
1392                         device_printf(dev, "ixl_setup_queue_msix() error: %d\n",
1393                             error);
1394                 error = ixl_setup_queue_tqs(vsi);
1395                 if (error)
1396                         device_printf(dev, "ixl_setup_queue_tqs() error: %d\n",
1397                             error);
1398         } else
1399                 // possibly broken
1400                 error = ixl_assign_vsi_legacy(pf);
1401         if (error) {
1402                 device_printf(pf->dev, "assign_vsi_msix/legacy error: %d\n", error);
1403                 return;
1404         }
1405
1406         IXL_PF_LOCK(pf);
1407         ixl_init_locked(pf);
1408         IXL_PF_UNLOCK(pf);
1409 }
1410
1411 /*
1412 ** MSIX Interrupt Handlers and Tasklets
1413 */
1414 static void
1415 ixl_handle_que(void *context, int pending)
1416 {
1417         struct ixl_queue *que = context;
1418         struct ixl_vsi *vsi = que->vsi;
1419         struct i40e_hw  *hw = vsi->hw;
1420         struct tx_ring  *txr = &que->txr;
1421         struct ifnet    *ifp = vsi->ifp;
1422         bool            more;
1423
1424         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1425                 more = ixl_rxeof(que, IXL_RX_LIMIT);
1426                 IXL_TX_LOCK(txr);
1427                 ixl_txeof(que);
1428                 if (!drbr_empty(ifp, txr->br))
1429                         ixl_mq_start_locked(ifp, txr);
1430                 IXL_TX_UNLOCK(txr);
1431                 if (more) {
1432                         taskqueue_enqueue(que->tq, &que->task);
1433                         return;
1434                 }
1435         }
1436
1437         /* Reenable this interrupt - hmmm */
1438         ixl_enable_queue(hw, que->me);
1439         return;
1440 }
1441
1442
1443 /*********************************************************************
1444  *
1445  *  Legacy Interrupt Service routine
1446  *
1447  **********************************************************************/
1448 void
1449 ixl_intr(void *arg)
1450 {
1451         struct ixl_pf           *pf = arg;
1452         struct i40e_hw          *hw =  &pf->hw;
1453         struct ixl_vsi          *vsi = &pf->vsi;
1454         struct ixl_queue        *que = vsi->queues;
1455         struct ifnet            *ifp = vsi->ifp;
1456         struct tx_ring          *txr = &que->txr;
1457         u32                     reg, icr0, mask;
1458         bool                    more_tx, more_rx;
1459
1460         ++que->irqs;
1461
1462         /* Protect against spurious interrupts */
1463         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1464                 return;
1465
1466         icr0 = rd32(hw, I40E_PFINT_ICR0);
1467
1468         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1469         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1470         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1471
1472         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1473
1474 #ifdef PCI_IOV
1475         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1476                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1477 #endif
1478
1479         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1480                 taskqueue_enqueue(pf->tq, &pf->adminq);
1481                 return;
1482         }
1483
1484         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1485
1486         IXL_TX_LOCK(txr);
1487         more_tx = ixl_txeof(que);
1488         if (!drbr_empty(vsi->ifp, txr->br))
1489                 more_tx = 1;
1490         IXL_TX_UNLOCK(txr);
1491
1492         /* re-enable other interrupt causes */
1493         wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1494
1495         /* And now the queues */
1496         reg = rd32(hw, I40E_QINT_RQCTL(0));
1497         reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1498         wr32(hw, I40E_QINT_RQCTL(0), reg);
1499
1500         reg = rd32(hw, I40E_QINT_TQCTL(0));
1501         reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1502         reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1503         wr32(hw, I40E_QINT_TQCTL(0), reg);
1504
1505         ixl_enable_legacy(hw);
1506
1507         return;
1508 }
1509
1510
1511 /*********************************************************************
1512  *
1513  *  MSIX VSI Interrupt Service routine
1514  *
1515  **********************************************************************/
1516 void
1517 ixl_msix_que(void *arg)
1518 {
1519         struct ixl_queue        *que = arg;
1520         struct ixl_vsi  *vsi = que->vsi;
1521         struct i40e_hw  *hw = vsi->hw;
1522         struct tx_ring  *txr = &que->txr;
1523         bool            more_tx, more_rx;
1524
1525         /* Protect against spurious interrupts */
1526         if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1527                 return;
1528
1529         ++que->irqs;
1530
1531         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1532
1533         IXL_TX_LOCK(txr);
1534         more_tx = ixl_txeof(que);
1535         /*
1536         ** Make certain that if the stack
1537         ** has anything queued the task gets
1538         ** scheduled to handle it.
1539         */
1540         if (!drbr_empty(vsi->ifp, txr->br))
1541                 more_tx = 1;
1542         IXL_TX_UNLOCK(txr);
1543
1544         ixl_set_queue_rx_itr(que);
1545         ixl_set_queue_tx_itr(que);
1546
1547         if (more_tx || more_rx)
1548                 taskqueue_enqueue(que->tq, &que->task);
1549         else
1550                 ixl_enable_queue(hw, que->me);
1551
1552         return;
1553 }
1554
1555
1556 /*********************************************************************
1557  *
1558  *  MSIX Admin Queue Interrupt Service routine
1559  *
1560  **********************************************************************/
1561 static void
1562 ixl_msix_adminq(void *arg)
1563 {
1564         struct ixl_pf   *pf = arg;
1565         struct i40e_hw  *hw = &pf->hw;
1566         u32             reg, mask, rstat_reg;
1567         bool            do_task = FALSE;
1568
1569         ++pf->admin_irq;
1570
1571         reg = rd32(hw, I40E_PFINT_ICR0);
1572         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1573
1574         /* Check on the cause */
1575         if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) {
1576                 mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK;
1577                 do_task = TRUE;
1578         }
1579
1580         if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1581                 ixl_handle_mdd_event(pf);
1582                 mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK;
1583         }
1584
1585         if (reg & I40E_PFINT_ICR0_GRST_MASK) {
1586                 device_printf(pf->dev, "Reset Requested!\n");
1587                 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT);
1588                 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
1589                     >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
1590                 device_printf(pf->dev, "Reset type: ");
1591                 switch (rstat_reg) {
1592                 /* These others might be handled similarly to an EMPR reset */
1593                 case I40E_RESET_CORER:
1594                         printf("CORER\n");
1595                         break;
1596                 case I40E_RESET_GLOBR:
1597                         printf("GLOBR\n");
1598                         break;
1599                 case I40E_RESET_EMPR:
1600                         printf("EMPR\n");
1601                         atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
1602                         break;
1603                 default:
1604                         printf("?\n");
1605                         break;
1606                 }
1607                 // overload admin queue task to check reset progress?
1608                 do_task = TRUE;
1609         }
1610
1611         if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) {
1612                 device_printf(pf->dev, "ECC Error detected!\n");
1613         }
1614
1615         if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) {
1616                 device_printf(pf->dev, "HMC Error detected!\n");
1617         }
1618
1619         if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) {
1620                 device_printf(pf->dev, "PCI Exception detected!\n");
1621         }
1622
1623 #ifdef PCI_IOV
1624         if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1625                 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1626                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1627         }
1628 #endif
1629
1630         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1631         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1632         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1633
1634         if (do_task)
1635                 taskqueue_enqueue(pf->tq, &pf->adminq);
1636 }
1637
1638 /*********************************************************************
1639  *
1640  *  Media Ioctl callback
1641  *
1642  *  This routine is called whenever the user queries the status of
1643  *  the interface using ifconfig.
1644  *
1645  **********************************************************************/
1646 static void
1647 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1648 {
1649         struct ixl_vsi  *vsi = ifp->if_softc;
1650         struct ixl_pf   *pf = vsi->back;
1651         struct i40e_hw  *hw = &pf->hw;
1652
1653         INIT_DEBUGOUT("ixl_media_status: begin");
1654         IXL_PF_LOCK(pf);
1655
1656         hw->phy.get_link_info = TRUE;
1657         i40e_get_link_status(hw, &pf->link_up);
1658         ixl_update_link_status(pf);
1659
1660         ifmr->ifm_status = IFM_AVALID;
1661         ifmr->ifm_active = IFM_ETHER;
1662
1663         if (!pf->link_up) {
1664                 IXL_PF_UNLOCK(pf);
1665                 return;
1666         }
1667
1668         ifmr->ifm_status |= IFM_ACTIVE;
1669
1670         /* Hardware always does full-duplex */
1671         ifmr->ifm_active |= IFM_FDX;
1672
1673         switch (hw->phy.link_info.phy_type) {
1674                 /* 100 M */
1675                 case I40E_PHY_TYPE_100BASE_TX:
1676                         ifmr->ifm_active |= IFM_100_TX;
1677                         break;
1678                 /* 1 G */
1679                 case I40E_PHY_TYPE_1000BASE_T:
1680                         ifmr->ifm_active |= IFM_1000_T;
1681                         break;
1682                 case I40E_PHY_TYPE_1000BASE_SX:
1683                         ifmr->ifm_active |= IFM_1000_SX;
1684                         break;
1685                 case I40E_PHY_TYPE_1000BASE_LX:
1686                         ifmr->ifm_active |= IFM_1000_LX;
1687                         break;
1688                 case I40E_PHY_TYPE_1000BASE_T_OPTICAL:
1689                         ifmr->ifm_active |= IFM_OTHER;
1690                         break;
1691                 /* 10 G */
1692                 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1693                         ifmr->ifm_active |= IFM_10G_TWINAX;
1694                         break;
1695                 case I40E_PHY_TYPE_10GBASE_SR:
1696                         ifmr->ifm_active |= IFM_10G_SR;
1697                         break;
1698                 case I40E_PHY_TYPE_10GBASE_LR:
1699                         ifmr->ifm_active |= IFM_10G_LR;
1700                         break;
1701                 case I40E_PHY_TYPE_10GBASE_T:
1702                         ifmr->ifm_active |= IFM_10G_T;
1703                         break;
1704                 case I40E_PHY_TYPE_XAUI:
1705                 case I40E_PHY_TYPE_XFI:
1706                 case I40E_PHY_TYPE_10GBASE_AOC:
1707                         ifmr->ifm_active |= IFM_OTHER;
1708                         break;
1709                 /* 40 G */
1710                 case I40E_PHY_TYPE_40GBASE_CR4:
1711                 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1712                         ifmr->ifm_active |= IFM_40G_CR4;
1713                         break;
1714                 case I40E_PHY_TYPE_40GBASE_SR4:
1715                         ifmr->ifm_active |= IFM_40G_SR4;
1716                         break;
1717                 case I40E_PHY_TYPE_40GBASE_LR4:
1718                         ifmr->ifm_active |= IFM_40G_LR4;
1719                         break;
1720                 case I40E_PHY_TYPE_XLAUI:
1721                         ifmr->ifm_active |= IFM_OTHER;
1722                         break;
1723 #ifndef IFM_ETH_XTYPE
1724                 case I40E_PHY_TYPE_1000BASE_KX:
1725                         ifmr->ifm_active |= IFM_1000_CX;
1726                         break;
1727                 case I40E_PHY_TYPE_SGMII:
1728                         ifmr->ifm_active |= IFM_OTHER;
1729                         break;
1730                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1731                 case I40E_PHY_TYPE_10GBASE_CR1:
1732                         ifmr->ifm_active |= IFM_10G_TWINAX;
1733                         break;
1734                 case I40E_PHY_TYPE_10GBASE_KX4:
1735                         ifmr->ifm_active |= IFM_10G_CX4;
1736                         break;
1737                 case I40E_PHY_TYPE_10GBASE_KR:
1738                         ifmr->ifm_active |= IFM_10G_SR;
1739                         break;
1740                 case I40E_PHY_TYPE_SFI:
1741                         ifmr->ifm_active |= IFM_OTHER;
1742                         break;
1743                 case I40E_PHY_TYPE_40GBASE_KR4:
1744                 case I40E_PHY_TYPE_XLPPI:
1745                 case I40E_PHY_TYPE_40GBASE_AOC:
1746                         ifmr->ifm_active |= IFM_40G_SR4;
1747                         break;
1748 #else
1749                 case I40E_PHY_TYPE_1000BASE_KX:
1750                         ifmr->ifm_active |= IFM_1000_KX;
1751                         break;
1752                 case I40E_PHY_TYPE_SGMII:
1753                         ifmr->ifm_active |= IFM_1000_SGMII;
1754                         break;
1755                 /* ERJ: What's the difference between these? */
1756                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1757                 case I40E_PHY_TYPE_10GBASE_CR1:
1758                         ifmr->ifm_active |= IFM_10G_CR1;
1759                         break;
1760                 case I40E_PHY_TYPE_10GBASE_KX4:
1761                         ifmr->ifm_active |= IFM_10G_KX4;
1762                         break;
1763                 case I40E_PHY_TYPE_10GBASE_KR:
1764                         ifmr->ifm_active |= IFM_10G_KR;
1765                         break;
1766                 case I40E_PHY_TYPE_SFI:
1767                         ifmr->ifm_active |= IFM_10G_SFI;
1768                         break;
1769                 /* Our single 20G media type */
1770                 case I40E_PHY_TYPE_20GBASE_KR2:
1771                         ifmr->ifm_active |= IFM_20G_KR2;
1772                         break;
1773                 case I40E_PHY_TYPE_40GBASE_KR4:
1774                         ifmr->ifm_active |= IFM_40G_KR4;
1775                         break;
1776                 case I40E_PHY_TYPE_XLPPI:
1777                 case I40E_PHY_TYPE_40GBASE_AOC:
1778                         ifmr->ifm_active |= IFM_40G_XLPPI;
1779                         break;
1780 #endif
1781                 /* Unknown to driver */
1782                 default:
1783                         ifmr->ifm_active |= IFM_UNKNOWN;
1784                         break;
1785         }
1786         /* Report flow control status as well */
1787         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1788                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1789         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1790                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1791
1792         IXL_PF_UNLOCK(pf);
1793
1794         return;
1795 }
1796
1797 /*
1798  * NOTE: Fortville does not support forcing media speeds. Instead,
1799  * use the set_advertise sysctl to set the speeds Fortville
1800  * will advertise or be allowed to operate at.
1801  */
1802 static int
1803 ixl_media_change(struct ifnet * ifp)
1804 {
1805         struct ixl_vsi *vsi = ifp->if_softc;
1806         struct ifmedia *ifm = &vsi->media;
1807
1808         INIT_DEBUGOUT("ixl_media_change: begin");
1809
1810         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1811                 return (EINVAL);
1812
1813         if_printf(ifp, "Media change is not supported.\n");
1814
1815         return (ENODEV);
1816 }
1817
1818
1819 #ifdef IXL_FDIR
1820 /*
1821 ** ATR: Application Targetted Receive - creates a filter
1822 **      based on TX flow info that will keep the receive
1823 **      portion of the flow on the same queue. Based on the
1824 **      implementation this is only available for TCP connections
1825 */
1826 void
1827 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1828 {
1829         struct ixl_vsi                  *vsi = que->vsi;
1830         struct tx_ring                  *txr = &que->txr;
1831         struct i40e_filter_program_desc *FDIR;
1832         u32                             ptype, dtype;
1833         int                             idx;
1834
1835         /* check if ATR is enabled and sample rate */
1836         if ((!ixl_enable_fdir) || (!txr->atr_rate))
1837                 return;
1838         /*
1839         ** We sample all TCP SYN/FIN packets,
1840         ** or at the selected sample rate
1841         */
1842         txr->atr_count++;
1843         if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1844             (txr->atr_count < txr->atr_rate))
1845                 return;
1846         txr->atr_count = 0;
1847
1848         /* Get a descriptor to use */
1849         idx = txr->next_avail;
1850         FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1851         if (++idx == que->num_desc)
1852                 idx = 0;
1853         txr->avail--;
1854         txr->next_avail = idx;
1855
1856         ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1857             I40E_TXD_FLTR_QW0_QINDEX_MASK;
1858
1859         ptype |= (etype == ETHERTYPE_IP) ?
1860             (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1861             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1862             (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1863             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1864
1865         ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1866
1867         dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1868
1869         /*
1870         ** We use the TCP TH_FIN as a trigger to remove
1871         ** the filter, otherwise its an update.
1872         */
1873         dtype |= (th->th_flags & TH_FIN) ?
1874             (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1875             I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1876             (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1877             I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1878
1879         dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1880             I40E_TXD_FLTR_QW1_DEST_SHIFT;
1881
1882         dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1883             I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1884
1885         FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1886         FDIR->dtype_cmd_cntindex = htole32(dtype);
1887         return;
1888 }
1889 #endif
1890
1891
1892 static void
1893 ixl_set_promisc(struct ixl_vsi *vsi)
1894 {
1895         struct ifnet    *ifp = vsi->ifp;
1896         struct i40e_hw  *hw = vsi->hw;
1897         int             err, mcnt = 0;
1898         bool            uni = FALSE, multi = FALSE;
1899
1900         if (ifp->if_flags & IFF_ALLMULTI)
1901                 multi = TRUE;
1902         else { /* Need to count the multicast addresses */
1903                 struct  ifmultiaddr *ifma;
1904                 if_maddr_rlock(ifp);
1905                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1906                         if (ifma->ifma_addr->sa_family != AF_LINK)
1907                                 continue;
1908                         if (mcnt == MAX_MULTICAST_ADDR)
1909                                 break;
1910                         mcnt++;
1911                 }
1912                 if_maddr_runlock(ifp);
1913         }
1914
1915         if (mcnt >= MAX_MULTICAST_ADDR)
1916                 multi = TRUE;
1917         if (ifp->if_flags & IFF_PROMISC)
1918                 uni = TRUE;
1919
1920         err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1921             vsi->seid, uni, NULL);
1922         err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1923             vsi->seid, multi, NULL);
1924         return;
1925 }
1926
1927 /*********************************************************************
1928  *      Filter Routines
1929  *
1930  *      Routines for multicast and vlan filter management.
1931  *
1932  *********************************************************************/
1933 static void
1934 ixl_add_multi(struct ixl_vsi *vsi)
1935 {
1936         struct  ifmultiaddr     *ifma;
1937         struct ifnet            *ifp = vsi->ifp;
1938         struct i40e_hw          *hw = vsi->hw;
1939         int                     mcnt = 0, flags;
1940
1941         IOCTL_DEBUGOUT("ixl_add_multi: begin");
1942
1943         if_maddr_rlock(ifp);
1944         /*
1945         ** First just get a count, to decide if we
1946         ** we simply use multicast promiscuous.
1947         */
1948         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1949                 if (ifma->ifma_addr->sa_family != AF_LINK)
1950                         continue;
1951                 mcnt++;
1952         }
1953         if_maddr_runlock(ifp);
1954
1955         if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1956                 /* delete existing MC filters */
1957                 ixl_del_hw_filters(vsi, mcnt);
1958                 i40e_aq_set_vsi_multicast_promiscuous(hw,
1959                     vsi->seid, TRUE, NULL);
1960                 return;
1961         }
1962
1963         mcnt = 0;
1964         if_maddr_rlock(ifp);
1965         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1966                 if (ifma->ifma_addr->sa_family != AF_LINK)
1967                         continue;
1968                 ixl_add_mc_filter(vsi,
1969                     (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1970                 mcnt++;
1971         }
1972         if_maddr_runlock(ifp);
1973         if (mcnt > 0) {
1974                 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1975                 ixl_add_hw_filters(vsi, flags, mcnt);
1976         }
1977
1978         IOCTL_DEBUGOUT("ixl_add_multi: end");
1979         return;
1980 }
1981
1982 static void
1983 ixl_del_multi(struct ixl_vsi *vsi)
1984 {
1985         struct ifnet            *ifp = vsi->ifp;
1986         struct ifmultiaddr      *ifma;
1987         struct ixl_mac_filter   *f;
1988         int                     mcnt = 0;
1989         bool            match = FALSE;
1990
1991         IOCTL_DEBUGOUT("ixl_del_multi: begin");
1992
1993         /* Search for removed multicast addresses */
1994         if_maddr_rlock(ifp);
1995         SLIST_FOREACH(f, &vsi->ftl, next) {
1996                 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1997                         match = FALSE;
1998                         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1999                                 if (ifma->ifma_addr->sa_family != AF_LINK)
2000                                         continue;
2001                                 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2002                                 if (cmp_etheraddr(f->macaddr, mc_addr)) {
2003                                         match = TRUE;
2004                                         break;
2005                                 }
2006                         }
2007                         if (match == FALSE) {
2008                                 f->flags |= IXL_FILTER_DEL;
2009                                 mcnt++;
2010                         }
2011                 }
2012         }
2013         if_maddr_runlock(ifp);
2014
2015         if (mcnt > 0)
2016                 ixl_del_hw_filters(vsi, mcnt);
2017 }
2018
2019
2020 /*********************************************************************
2021  *  Timer routine
2022  *
2023  *  This routine checks for link status,updates statistics,
2024  *  and runs the watchdog check.
2025  *
2026  *  Only runs when the driver is configured UP and RUNNING.
2027  *
2028  **********************************************************************/
2029
2030 static void
2031 ixl_local_timer(void *arg)
2032 {
2033         struct ixl_pf           *pf = arg;
2034         struct i40e_hw          *hw = &pf->hw;
2035         struct ixl_vsi          *vsi = &pf->vsi;
2036         struct ixl_queue        *que = vsi->queues;
2037         device_t                dev = pf->dev;
2038         int                     hung = 0;
2039         u32                     mask;
2040
2041         mtx_assert(&pf->pf_mtx, MA_OWNED);
2042
2043         /* Fire off the adminq task */
2044         taskqueue_enqueue(pf->tq, &pf->adminq);
2045
2046         /* Update stats */
2047         ixl_update_stats_counters(pf);
2048
2049         /*
2050         ** Check status of the queues
2051         */
2052         mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
2053                 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
2054
2055         for (int i = 0; i < vsi->num_queues; i++, que++) {
2056                 /* Any queues with outstanding work get a sw irq */
2057                 if (que->busy)
2058                         wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
2059                 /*
2060                 ** Each time txeof runs without cleaning, but there
2061                 ** are uncleaned descriptors it increments busy. If
2062                 ** we get to 5 we declare it hung.
2063                 */
2064                 if (que->busy == IXL_QUEUE_HUNG) {
2065                         ++hung;
2066                         /* Mark the queue as inactive */
2067                         vsi->active_queues &= ~((u64)1 << que->me);
2068                         continue;
2069                 } else {
2070                         /* Check if we've come back from hung */
2071                         if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
2072                                 vsi->active_queues |= ((u64)1 << que->me);
2073                 }
2074                 if (que->busy >= IXL_MAX_TX_BUSY) {
2075 #ifdef IXL_DEBUG
2076                         device_printf(dev,"Warning queue %d "
2077                             "appears to be hung!\n", i);
2078 #endif
2079                         que->busy = IXL_QUEUE_HUNG;
2080                         ++hung;
2081                 }
2082         }
2083         /* Only reinit if all queues show hung */
2084         if (hung == vsi->num_queues)
2085                 goto hung;
2086
2087         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
2088         return;
2089
2090 hung:
2091         device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
2092         ixl_init_locked(pf);
2093 }
2094
2095 /*
2096 ** Note: this routine updates the OS on the link state
2097 **      the real check of the hardware only happens with
2098 **      a link interrupt.
2099 */
2100 static void
2101 ixl_update_link_status(struct ixl_pf *pf)
2102 {
2103         struct ixl_vsi          *vsi = &pf->vsi;
2104         struct i40e_hw          *hw = &pf->hw;
2105         struct ifnet            *ifp = vsi->ifp;
2106         device_t                dev = pf->dev;
2107
2108         if (pf->link_up) {
2109                 if (vsi->link_active == FALSE) {
2110                         pf->fc = hw->fc.current_mode;
2111                         if (bootverbose) {
2112                                 device_printf(dev,"Link is up %d Gbps %s,"
2113                                     " Flow Control: %s\n",
2114                                     ((pf->link_speed ==
2115                                     I40E_LINK_SPEED_40GB)? 40:10),
2116                                     "Full Duplex", ixl_fc_string[pf->fc]);
2117                         }
2118                         vsi->link_active = TRUE;
2119                         /*
2120                         ** Warn user if link speed on NPAR enabled
2121                         ** partition is not at least 10GB
2122                         */
2123                         if (hw->func_caps.npar_enable &&
2124                            (hw->phy.link_info.link_speed ==
2125                            I40E_LINK_SPEED_1GB ||
2126                            hw->phy.link_info.link_speed ==
2127                            I40E_LINK_SPEED_100MB))
2128                                 device_printf(dev, "The partition detected"
2129                                     "link speed that is less than 10Gbps\n");
2130                         if_link_state_change(ifp, LINK_STATE_UP);
2131                 }
2132         } else { /* Link down */
2133                 if (vsi->link_active == TRUE) {
2134                         if (bootverbose)
2135                                 device_printf(dev, "Link is Down\n");
2136                         if_link_state_change(ifp, LINK_STATE_DOWN);
2137                         vsi->link_active = FALSE;
2138                 }
2139         }
2140
2141         return;
2142 }
2143
2144 static void
2145 ixl_stop(struct ixl_pf *pf)
2146 {
2147         IXL_PF_LOCK(pf);
2148         ixl_stop_locked(pf);
2149         IXL_PF_UNLOCK(pf);
2150
2151         ixl_teardown_queue_msix(&pf->vsi);
2152         ixl_free_queue_tqs(&pf->vsi);
2153 }
2154
2155 /*********************************************************************
2156  *
2157  *  This routine disables all traffic on the adapter by issuing a
2158  *  global reset on the MAC and deallocates TX/RX buffers.
2159  *
2160  **********************************************************************/
2161
2162 static void
2163 ixl_stop_locked(struct ixl_pf *pf)
2164 {
2165         struct ixl_vsi  *vsi = &pf->vsi;
2166         struct ifnet    *ifp = vsi->ifp;
2167
2168         INIT_DEBUGOUT("ixl_stop: begin\n");
2169
2170         IXL_PF_LOCK_ASSERT(pf);
2171
2172         /* Stop the local timer */
2173         callout_stop(&pf->timer);
2174
2175         ixl_disable_rings_intr(vsi);
2176         ixl_disable_rings(vsi);
2177
2178         /* Tell the stack that the interface is no longer active */
2179         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING);
2180 }
2181
2182
2183 /*********************************************************************
2184  *
2185  *  Setup MSIX Interrupt resources and handlers for the VSI
2186  *
2187  **********************************************************************/
2188 static int
2189 ixl_assign_vsi_legacy(struct ixl_pf *pf)
2190 {
2191         device_t        dev = pf->dev;
2192         struct          ixl_vsi *vsi = &pf->vsi;
2193         struct          ixl_queue *que = vsi->queues;
2194         int             error, rid = 0;
2195
2196         if (pf->msix == 1)
2197                 rid = 1;
2198         pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2199             &rid, RF_SHAREABLE | RF_ACTIVE);
2200         if (pf->res == NULL) {
2201                 device_printf(dev, "Unable to allocate"
2202                     " bus resource: vsi legacy/msi interrupt\n");
2203                 return (ENXIO);
2204         }
2205
2206         /* Set the handler function */
2207         error = bus_setup_intr(dev, pf->res,
2208             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2209             ixl_intr, pf, &pf->tag);
2210         if (error) {
2211                 pf->res = NULL;
2212                 device_printf(dev, "Failed to register legacy/msi handler\n");
2213                 return (error);
2214         }
2215         bus_describe_intr(dev, pf->res, pf->tag, "irq0");
2216         TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2217         TASK_INIT(&que->task, 0, ixl_handle_que, que);
2218         que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2219             taskqueue_thread_enqueue, &que->tq);
2220         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2221             device_get_nameunit(dev));
2222         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2223
2224         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2225             taskqueue_thread_enqueue, &pf->tq);
2226         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2227             device_get_nameunit(dev));
2228
2229         return (0);
2230 }
2231
2232 static int
2233 ixl_setup_adminq_tq(struct ixl_pf *pf)
2234 {
2235         device_t dev = pf->dev;
2236         int error = 0;
2237
2238         /* Tasklet for Admin Queue interrupts */
2239         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2240 #ifdef PCI_IOV
2241         /* VFLR Tasklet */
2242         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2243 #endif
2244         /* Create and start Admin Queue taskqueue */
2245         pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT,
2246             taskqueue_thread_enqueue, &pf->tq);
2247         if (!pf->tq) {
2248                 device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n");
2249                 return (ENOMEM);
2250         }
2251         error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq",
2252             device_get_nameunit(dev));
2253         if (error) {
2254                 device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n",
2255                     error);
2256                 taskqueue_free(pf->tq);
2257                 return (error);
2258         }
2259         return (0);
2260 }
2261
2262 static int
2263 ixl_setup_queue_tqs(struct ixl_vsi *vsi)
2264 {
2265         struct ixl_queue *que = vsi->queues;
2266         device_t dev = vsi->dev;
2267 #ifdef  RSS
2268         cpuset_t cpu_mask;
2269         int cpu_id;
2270 #endif
2271
2272         /* Create queue tasks and start queue taskqueues */
2273         for (int i = 0; i < vsi->num_queues; i++, que++) {
2274                 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2275                 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2276                 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2277                     taskqueue_thread_enqueue, &que->tq);
2278 #ifdef RSS
2279                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2280                 CPU_SETOF(cpu_id, &cpu_mask);
2281                 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2282                     &cpu_mask, "%s (bucket %d)",
2283                     device_get_nameunit(dev), cpu_id);
2284 #else
2285                 taskqueue_start_threads(&que->tq, 1, PI_NET,
2286                     "%s (que %d)", device_get_nameunit(dev), que->me);
2287 #endif
2288         }
2289
2290         return (0);
2291 }
2292
2293 static void
2294 ixl_free_adminq_tq(struct ixl_pf *pf)
2295 {
2296         if (pf->tq) {
2297                 taskqueue_free(pf->tq);
2298                 pf->tq = NULL;
2299         }
2300 }
2301
2302 static void
2303 ixl_free_queue_tqs(struct ixl_vsi *vsi)
2304 {
2305         struct ixl_queue *que = vsi->queues;
2306
2307         for (int i = 0; i < vsi->num_queues; i++, que++) {
2308                 if (que->tq) {
2309                         taskqueue_free(que->tq);
2310                         que->tq = NULL;
2311                 }
2312         }
2313 }
2314
2315 static int
2316 ixl_setup_adminq_msix(struct ixl_pf *pf)
2317 {
2318         device_t dev = pf->dev;
2319         int rid, error = 0;
2320
2321         /* Admin IRQ rid is 1, vector is 0 */
2322         rid = 1;
2323         /* Get interrupt resource from bus */
2324         pf->res = bus_alloc_resource_any(dev,
2325             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2326         if (!pf->res) {
2327                 device_printf(dev, "bus_alloc_resource_any() for Admin Queue"
2328                     " interrupt failed [rid=%d]\n", rid);
2329                 return (ENXIO);
2330         }
2331         /* Then associate interrupt with handler */
2332         error = bus_setup_intr(dev, pf->res,
2333             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2334             ixl_msix_adminq, pf, &pf->tag);
2335         if (error) {
2336                 pf->res = NULL;
2337                 device_printf(dev, "bus_setup_intr() for Admin Queue"
2338                     " interrupt handler failed, error %d\n", error);
2339                 return (ENXIO);
2340         }
2341         error = bus_describe_intr(dev, pf->res, pf->tag, "aq");
2342         if (error) {
2343                 /* Probably non-fatal? */
2344                 device_printf(dev, "bus_describe_intr() for Admin Queue"
2345                     " interrupt name failed, error %d\n", error);
2346         }
2347         pf->admvec = 0;
2348
2349         return (0);
2350 }
2351
2352 /*
2353  * Allocate interrupt resources from bus and associate an interrupt handler
2354  * to those for the VSI's queues.
2355  */
2356 static int
2357 ixl_setup_queue_msix(struct ixl_vsi *vsi)
2358 {
2359         device_t        dev = vsi->dev;
2360         struct          ixl_queue *que = vsi->queues;
2361         struct          tx_ring  *txr;
2362         int             error, rid, vector = 1;
2363
2364         /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */
2365         for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2366                 int cpu_id = i;
2367                 rid = vector + 1;
2368                 txr = &que->txr;
2369                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2370                     RF_SHAREABLE | RF_ACTIVE);
2371                 if (!que->res) {
2372                         device_printf(dev, "bus_alloc_resource_any() for"
2373                             " Queue %d interrupt failed [rid=%d]\n",
2374                             que->me, rid);
2375                         return (ENXIO);
2376                 }
2377                 /* Set the handler function */
2378                 error = bus_setup_intr(dev, que->res,
2379                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2380                     ixl_msix_que, que, &que->tag);
2381                 if (error) {
2382                         device_printf(dev, "bus_setup_intr() for Queue %d"
2383                             " interrupt handler failed, error %d\n",
2384                             que->me, error);
2385                         // TODO: Check for error from this?
2386                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2387                         return (error);
2388                 }
2389                 error = bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2390                 if (error) {
2391                         device_printf(dev, "bus_describe_intr() for Queue %d"
2392                             " interrupt name failed, error %d\n",
2393                             que->me, error);
2394                 }
2395                 /* Bind the vector to a CPU */
2396 #ifdef RSS
2397                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2398 #endif
2399                 error = bus_bind_intr(dev, que->res, cpu_id);
2400                 if (error) {
2401                         device_printf(dev, "bus_bind_intr() for Queue %d"
2402                             " to CPU %d failed, error %d\n",
2403                             que->me, cpu_id, error);
2404                 }
2405                 que->msix = vector;
2406         }
2407
2408         return (0);
2409 }
2410
2411
2412 /*
2413  * Allocate MSI/X vectors
2414  */
2415 static int
2416 ixl_init_msix(struct ixl_pf *pf)
2417 {
2418         device_t dev = pf->dev;
2419         int rid, want, vectors, queues, available;
2420
2421         /* Override by tuneable */
2422         if (ixl_enable_msix == 0)
2423                 goto no_msix;
2424
2425         /*
2426         ** When used in a virtualized environment
2427         ** PCI BUSMASTER capability may not be set
2428         ** so explicity set it here and rewrite
2429         ** the ENABLE in the MSIX control register
2430         ** at this point to cause the host to
2431         ** successfully initialize us.
2432         */
2433         {
2434                 u16 pci_cmd_word;
2435                 int msix_ctrl;
2436                 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2437                 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2438                 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2439                 pci_find_cap(dev, PCIY_MSIX, &rid);
2440                 rid += PCIR_MSIX_CTRL;
2441                 msix_ctrl = pci_read_config(dev, rid, 2);
2442                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2443                 pci_write_config(dev, rid, msix_ctrl, 2);
2444         }
2445
2446         /* First try MSI/X */
2447         rid = PCIR_BAR(IXL_BAR);
2448         pf->msix_mem = bus_alloc_resource_any(dev,
2449             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2450         if (!pf->msix_mem) {
2451                 /* May not be enabled */
2452                 device_printf(pf->dev,
2453                     "Unable to map MSIX table\n");
2454                 goto no_msix;
2455         }
2456
2457         available = pci_msix_count(dev);
2458         if (available == 0) { /* system has msix disabled */
2459                 bus_release_resource(dev, SYS_RES_MEMORY,
2460                     rid, pf->msix_mem);
2461                 pf->msix_mem = NULL;
2462                 goto no_msix;
2463         }
2464
2465         /* Figure out a reasonable auto config value */
2466         queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2467
2468         /* Override with tunable value if tunable is less than autoconfig count */
2469         if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2470                 queues = ixl_max_queues;
2471         else if ((ixl_max_queues != 0) && (ixl_max_queues > queues))
2472                 device_printf(dev, "ixl_max_queues > # of cpus, using "
2473                     "autoconfig amount...\n");
2474         /* Or limit maximum auto-configured queues to 8 */
2475         else if ((ixl_max_queues == 0) && (queues > 8))
2476                 queues = 8;
2477
2478 #ifdef  RSS
2479         /* If we're doing RSS, clamp at the number of RSS buckets */
2480         if (queues > rss_getnumbuckets())
2481                 queues = rss_getnumbuckets();
2482 #endif
2483
2484         /*
2485         ** Want one vector (RX/TX pair) per queue
2486         ** plus an additional for the admin queue.
2487         */
2488         want = queues + 1;
2489         if (want <= available)  /* Have enough */
2490                 vectors = want;
2491         else {
2492                 device_printf(pf->dev,
2493                     "MSIX Configuration Problem, "
2494                     "%d vectors available but %d wanted!\n",
2495                     available, want);
2496                 return (0); /* Will go to Legacy setup */
2497         }
2498
2499         if (pci_alloc_msix(dev, &vectors) == 0) {
2500                 device_printf(pf->dev,
2501                     "Using MSIX interrupts with %d vectors\n", vectors);
2502                 pf->msix = vectors;
2503                 pf->vsi.num_queues = queues;
2504 #ifdef RSS
2505                 /*
2506                  * If we're doing RSS, the number of queues needs to
2507                  * match the number of RSS buckets that are configured.
2508                  *
2509                  * + If there's more queues than RSS buckets, we'll end
2510                  *   up with queues that get no traffic.
2511                  *
2512                  * + If there's more RSS buckets than queues, we'll end
2513                  *   up having multiple RSS buckets map to the same queue,
2514                  *   so there'll be some contention.
2515                  */
2516                 if (queues != rss_getnumbuckets()) {
2517                         device_printf(dev,
2518                             "%s: queues (%d) != RSS buckets (%d)"
2519                             "; performance will be impacted.\n",
2520                             __func__, queues, rss_getnumbuckets());
2521                 }
2522 #endif
2523                 return (vectors);
2524         }
2525 no_msix:
2526         vectors = pci_msi_count(dev);
2527         pf->vsi.num_queues = 1;
2528         ixl_max_queues = 1;
2529         ixl_enable_msix = 0;
2530         if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2531                 device_printf(pf->dev, "Using an MSI interrupt\n");
2532         else {
2533                 vectors = 0;
2534                 device_printf(pf->dev, "Using a Legacy interrupt\n");
2535         }
2536         return (vectors);
2537 }
2538
2539 /*
2540  * Configure admin queue/misc interrupt cause registers in hardware.
2541  */
2542 static void
2543 ixl_configure_intr0_msix(struct ixl_pf *pf)
2544 {
2545         struct i40e_hw *hw = &pf->hw;
2546         u32 reg;
2547
2548         /* First set up the adminq - vector 0 */
2549         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2550         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2551
2552         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2553             I40E_PFINT_ICR0_ENA_GRST_MASK |
2554             I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
2555             I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2556             I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2557             I40E_PFINT_ICR0_ENA_VFLR_MASK |
2558             I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2559         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2560
2561         /*
2562          * 0x7FF is the end of the queue list.
2563          * This means we won't use MSI-X vector 0 for a queue interrupt
2564          * in MSIX mode.
2565          */
2566         wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2567         /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */
2568         wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E);
2569
2570         wr32(hw, I40E_PFINT_DYN_CTL0,
2571             I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2572             I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2573
2574         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2575 }
2576
2577 /*
2578  * Configure queue interrupt cause registers in hardware.
2579  */
2580 static void
2581 ixl_configure_queue_intr_msix(struct ixl_pf *pf)
2582 {
2583         struct i40e_hw  *hw = &pf->hw;
2584         struct ixl_vsi *vsi = &pf->vsi;
2585         u32             reg;
2586         u16             vector = 1;
2587
2588         for (int i = 0; i < vsi->num_queues; i++, vector++) {
2589                 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
2590                 /* First queue type is RX / 0 */
2591                 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2592
2593                 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2594                 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2595                 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2596                 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2597                 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2598                 wr32(hw, I40E_QINT_RQCTL(i), reg);
2599
2600                 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2601                 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2602                 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2603                 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2604                 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2605                 wr32(hw, I40E_QINT_TQCTL(i), reg);
2606         }
2607 }
2608
2609 /*
2610  * Configure for MSI single vector operation
2611  */
2612 static void
2613 ixl_configure_legacy(struct ixl_pf *pf)
2614 {
2615         struct i40e_hw  *hw = &pf->hw;
2616         u32             reg;
2617
2618         wr32(hw, I40E_PFINT_ITR0(0), 0);
2619         wr32(hw, I40E_PFINT_ITR0(1), 0);
2620
2621         /* Setup "other" causes */
2622         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2623             | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2624             | I40E_PFINT_ICR0_ENA_GRST_MASK
2625             | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2626             | I40E_PFINT_ICR0_ENA_GPIO_MASK
2627             | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2628             | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2629             | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2630             | I40E_PFINT_ICR0_ENA_VFLR_MASK
2631             | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2632             ;
2633         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2634
2635         /* SW_ITR_IDX = 0, but don't change INTENA */
2636         wr32(hw, I40E_PFINT_DYN_CTL0,
2637             I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2638             I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2639         /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2640         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2641
2642         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2643         wr32(hw, I40E_PFINT_LNKLST0, 0);
2644
2645         /* Associate the queue pair to the vector and enable the q int */
2646         reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2647             | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2648             | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2649         wr32(hw, I40E_QINT_RQCTL(0), reg);
2650
2651         reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2652             | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2653             | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2654         wr32(hw, I40E_QINT_TQCTL(0), reg);
2655 }
2656
2657
2658 /*
2659  * Get initial ITR values from tunable values.
2660  */
2661 static void
2662 ixl_configure_itr(struct ixl_pf *pf)
2663 {
2664         struct i40e_hw          *hw = &pf->hw;
2665         struct ixl_vsi          *vsi = &pf->vsi;
2666         struct ixl_queue        *que = vsi->queues;
2667
2668         vsi->rx_itr_setting = ixl_rx_itr;
2669         vsi->tx_itr_setting = ixl_tx_itr;
2670
2671         for (int i = 0; i < vsi->num_queues; i++, que++) {
2672                 struct tx_ring  *txr = &que->txr;
2673                 struct rx_ring  *rxr = &que->rxr;
2674
2675                 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2676                     vsi->rx_itr_setting);
2677                 rxr->itr = vsi->rx_itr_setting;
2678                 rxr->latency = IXL_AVE_LATENCY;
2679
2680                 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2681                     vsi->tx_itr_setting);
2682                 txr->itr = vsi->tx_itr_setting;
2683                 txr->latency = IXL_AVE_LATENCY;
2684         }
2685 }
2686
2687
2688 static int
2689 ixl_allocate_pci_resources(struct ixl_pf *pf)
2690 {
2691         int             rid;
2692         device_t        dev = pf->dev;
2693
2694         rid = PCIR_BAR(0);
2695         pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2696             &rid, RF_ACTIVE);
2697
2698         if (!(pf->pci_mem)) {
2699                 device_printf(dev, "Unable to allocate bus resource: PCI memory\n");
2700                 return (ENXIO);
2701         }
2702
2703         pf->osdep.mem_bus_space_tag =
2704                 rman_get_bustag(pf->pci_mem);
2705         pf->osdep.mem_bus_space_handle =
2706                 rman_get_bushandle(pf->pci_mem);
2707         pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2708         pf->osdep.flush_reg = I40E_GLGEN_STAT;
2709         pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2710
2711         pf->hw.back = &pf->osdep;
2712
2713         /*
2714         ** Now setup MSI or MSI/X, should
2715         ** return us the number of supported
2716         ** vectors. (Will be 1 for MSI)
2717         */
2718         pf->msix = ixl_init_msix(pf);
2719         return (0);
2720 }
2721
2722 /*
2723  * Teardown and release the admin queue/misc vector
2724  * interrupt.
2725  */
2726 static int
2727 ixl_teardown_adminq_msix(struct ixl_pf *pf)
2728 {
2729         device_t                dev = pf->dev;
2730         int                     rid;
2731
2732         if (pf->admvec) /* we are doing MSIX */
2733                 rid = pf->admvec + 1;
2734         else
2735                 (pf->msix != 0) ? (rid = 1):(rid = 0);
2736
2737         // TODO: Check for errors from bus_teardown_intr
2738         // TODO: Check for errors from bus_release_resource
2739         if (pf->tag != NULL) {
2740                 bus_teardown_intr(dev, pf->res, pf->tag);
2741                 pf->tag = NULL;
2742         }
2743         if (pf->res != NULL) {
2744                 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2745                 pf->res = NULL;
2746         }
2747
2748         return (0);
2749 }
2750
2751 static int
2752 ixl_teardown_queue_msix(struct ixl_vsi *vsi)
2753 {
2754         struct ixl_queue        *que = vsi->queues;
2755         device_t                dev = vsi->dev;
2756         int                     rid, error = 0;
2757
2758         /* We may get here before stations are setup */
2759         if ((!ixl_enable_msix) || (que == NULL))
2760                 return (0);
2761
2762         /* Release all MSIX queue resources */
2763         for (int i = 0; i < vsi->num_queues; i++, que++) {
2764                 rid = que->msix + 1;
2765                 if (que->tag != NULL) {
2766                         error = bus_teardown_intr(dev, que->res, que->tag);
2767                         if (error) {
2768                                 device_printf(dev, "bus_teardown_intr() for"
2769                                     " Queue %d interrupt failed\n",
2770                                     que->me);
2771                                 // return (ENXIO);
2772                         }
2773                         que->tag = NULL;
2774                 }
2775                 if (que->res != NULL) {
2776                         error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2777                         if (error) {
2778                                 device_printf(dev, "bus_release_resource() for"
2779                                     " Queue %d interrupt failed [rid=%d]\n",
2780                                     que->me, rid);
2781                                 // return (ENXIO);
2782                         }
2783                         que->res = NULL;
2784                 }
2785         }
2786
2787         return (0);
2788 }
2789
2790 static void
2791 ixl_free_pci_resources(struct ixl_pf *pf)
2792 {
2793         device_t                dev = pf->dev;
2794         int                     memrid;
2795
2796         ixl_teardown_queue_msix(&pf->vsi);
2797         ixl_teardown_adminq_msix(pf);
2798
2799         if (pf->msix)
2800                 pci_release_msi(dev);
2801
2802         memrid = PCIR_BAR(IXL_BAR);
2803
2804         if (pf->msix_mem != NULL)
2805                 bus_release_resource(dev, SYS_RES_MEMORY,
2806                     memrid, pf->msix_mem);
2807
2808         if (pf->pci_mem != NULL)
2809                 bus_release_resource(dev, SYS_RES_MEMORY,
2810                     PCIR_BAR(0), pf->pci_mem);
2811
2812         return;
2813 }
2814
2815 static void
2816 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2817 {
2818         /* Display supported media types */
2819         if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2820                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2821
2822         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2823                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2824         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2825                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2826         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2827                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2828
2829         if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2830             phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2831             phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2832                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2833
2834         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2835                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2836         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2837                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2838         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2839                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2840
2841         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2842             phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2843             phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2844             phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2845             phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2846                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2847         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2848                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2849         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2850                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2851
2852 #ifndef IFM_ETH_XTYPE
2853         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2854                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2855
2856         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2857             phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2858             phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2859             phy_type & (1 << I40E_PHY_TYPE_SFI))
2860                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2861         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2862                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2863         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2864                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2865
2866         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2867                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2868         if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2869                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2870 #else
2871         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2872                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2873
2874         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2875             || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2876                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2877         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2878                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2879         if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2880                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2881         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2882                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2883         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2884                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2885
2886         if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2887                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2888
2889         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2890                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2891         if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2892                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2893 #endif
2894 }
2895
2896 /*********************************************************************
2897  *
2898  *  Setup networking device structure and register an interface.
2899  *
2900  **********************************************************************/
2901 static int
2902 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2903 {
2904         struct ifnet            *ifp;
2905         struct i40e_hw          *hw = vsi->hw;
2906         struct ixl_queue        *que = vsi->queues;
2907         struct i40e_aq_get_phy_abilities_resp abilities;
2908         enum i40e_status_code aq_error = 0;
2909
2910         INIT_DEBUGOUT("ixl_setup_interface: begin");
2911
2912         ifp = vsi->ifp = if_alloc(IFT_ETHER);
2913         if (ifp == NULL) {
2914                 device_printf(dev, "can not allocate ifnet structure\n");
2915                 return (-1);
2916         }
2917         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2918         ifp->if_mtu = ETHERMTU;
2919         ifp->if_baudrate = IF_Gbps(40);
2920         ifp->if_init = ixl_init;
2921         ifp->if_softc = vsi;
2922         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2923         ifp->if_ioctl = ixl_ioctl;
2924
2925 #if __FreeBSD_version >= 1100036
2926         if_setgetcounterfn(ifp, ixl_get_counter);
2927 #endif
2928
2929         ifp->if_transmit = ixl_mq_start;
2930
2931         ifp->if_qflush = ixl_qflush;
2932
2933         ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2934
2935         vsi->max_frame_size =
2936             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2937             + ETHER_VLAN_ENCAP_LEN;
2938
2939         /*
2940          * Tell the upper layer(s) we support long frames.
2941          */
2942         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2943
2944         ifp->if_capabilities |= IFCAP_HWCSUM;
2945         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2946         ifp->if_capabilities |= IFCAP_TSO;
2947         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2948         ifp->if_capabilities |= IFCAP_LRO;
2949
2950         /* VLAN capabilties */
2951         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2952                              |  IFCAP_VLAN_HWTSO
2953                              |  IFCAP_VLAN_MTU
2954                              |  IFCAP_VLAN_HWCSUM;
2955         ifp->if_capenable = ifp->if_capabilities;
2956
2957         /*
2958         ** Don't turn this on by default, if vlans are
2959         ** created on another pseudo device (eg. lagg)
2960         ** then vlan events are not passed thru, breaking
2961         ** operation, but with HW FILTER off it works. If
2962         ** using vlans directly on the ixl driver you can
2963         ** enable this and get full hardware tag filtering.
2964         */
2965         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2966
2967         /*
2968          * Specify the media types supported by this adapter and register
2969          * callbacks to update media and link information
2970          */
2971         ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2972                      ixl_media_status);
2973
2974         aq_error = i40e_aq_get_phy_capabilities(hw,
2975             FALSE, TRUE, &abilities, NULL);
2976         /* May need delay to detect fiber correctly */
2977         if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2978                 i40e_msec_delay(200);
2979                 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2980                     TRUE, &abilities, NULL);
2981         }
2982         if (aq_error) {
2983                 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2984                         device_printf(dev, "Unknown PHY type detected!\n");
2985                 else
2986                         device_printf(dev,
2987                             "Error getting supported media types, err %d,"
2988                             " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2989                 return (0);
2990         }
2991
2992         ixl_add_ifmedia(vsi, abilities.phy_type);
2993
2994         /* Use autoselect media by default */
2995         ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2996         ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2997
2998         ether_ifattach(ifp, hw->mac.addr);
2999
3000         return (0);
3001 }
3002
3003 /*
3004 ** Run when the Admin Queue gets a link state change interrupt.
3005 */
3006 static void
3007 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
3008 {
3009         struct i40e_hw  *hw = &pf->hw;
3010         device_t dev = pf->dev;
3011         struct i40e_aqc_get_link_status *status =
3012             (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
3013
3014         /* Request link status from adapter */
3015         hw->phy.get_link_info = TRUE;
3016         i40e_get_link_status(hw, &pf->link_up);
3017
3018         /* Print out message if an unqualified module is found */
3019         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
3020             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
3021             (!(status->link_info & I40E_AQ_LINK_UP)))
3022                 device_printf(dev, "Link failed because "
3023                     "an unqualified module was detected!\n");
3024
3025         /* Update OS link info */
3026         ixl_update_link_status(pf);
3027 }
3028
3029 /*********************************************************************
3030  *
3031  *  Get Firmware Switch configuration
3032  *      - this will need to be more robust when more complex
3033  *        switch configurations are enabled.
3034  *
3035  **********************************************************************/
3036 static int
3037 ixl_switch_config(struct ixl_pf *pf)
3038 {
3039         struct i40e_hw  *hw = &pf->hw;
3040         struct ixl_vsi  *vsi = &pf->vsi;
3041         device_t        dev = vsi->dev;
3042         struct i40e_aqc_get_switch_config_resp *sw_config;
3043         u8      aq_buf[I40E_AQ_LARGE_BUF];
3044         int     ret;
3045         u16     next = 0;
3046
3047         memset(&aq_buf, 0, sizeof(aq_buf));
3048         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
3049         ret = i40e_aq_get_switch_config(hw, sw_config,
3050             sizeof(aq_buf), &next, NULL);
3051         if (ret) {
3052                 device_printf(dev, "aq_get_switch_config() failed, error %d,"
3053                     " aq_error %d\n", ret, pf->hw.aq.asq_last_status);
3054                 return (ret);
3055         }
3056 #ifdef IXL_DEBUG
3057         device_printf(dev,
3058             "Switch config: header reported: %d in structure, %d total\n",
3059             sw_config->header.num_reported, sw_config->header.num_total);
3060         for (int i = 0; i < sw_config->header.num_reported; i++) {
3061                 device_printf(dev,
3062                     "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
3063                     sw_config->element[i].element_type,
3064                     sw_config->element[i].seid,
3065                     sw_config->element[i].uplink_seid,
3066                     sw_config->element[i].downlink_seid);
3067         }
3068 #endif
3069         /* Simplified due to a single VSI at the moment */
3070         vsi->uplink_seid = sw_config->element[0].uplink_seid;
3071         vsi->downlink_seid = sw_config->element[0].downlink_seid;
3072         vsi->seid = sw_config->element[0].seid;
3073         return (ret);
3074 }
3075
3076 /*********************************************************************
3077  *
3078  *  Initialize the VSI:  this handles contexts, which means things
3079  *                       like the number of descriptors, buffer size,
3080  *                       plus we init the rings thru this function.
3081  *
3082  **********************************************************************/
3083 static int
3084 ixl_initialize_vsi(struct ixl_vsi *vsi)
3085 {
3086         struct ixl_pf           *pf = vsi->back;
3087         struct ixl_queue        *que = vsi->queues;
3088         device_t                dev = vsi->dev;
3089         struct i40e_hw          *hw = vsi->hw;
3090         struct i40e_vsi_context ctxt;
3091         int                     err = 0;
3092
3093         memset(&ctxt, 0, sizeof(ctxt));
3094         ctxt.seid = vsi->seid;
3095         if (pf->veb_seid != 0)
3096                 ctxt.uplink_seid = pf->veb_seid;
3097         ctxt.pf_num = hw->pf_id;
3098         err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
3099         if (err) {
3100                 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d"
3101                     " aq_error %d\n", err, hw->aq.asq_last_status);
3102                 return (err);
3103         }
3104 #ifdef IXL_DEBUG
3105         device_printf(dev, "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
3106             "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
3107             "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
3108             ctxt.uplink_seid, ctxt.vsi_number,
3109             ctxt.vsis_allocated, ctxt.vsis_unallocated,
3110             ctxt.flags, ctxt.pf_num, ctxt.vf_num,
3111             ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
3112 #endif
3113         /*
3114         ** Set the queue and traffic class bits
3115         **  - when multiple traffic classes are supported
3116         **    this will need to be more robust.
3117         */
3118         ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
3119         ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
3120         /* In contig mode, que_mapping[0] is first queue index used by this VSI */
3121         ctxt.info.queue_mapping[0] = 0;
3122         /*
3123          * This VSI will only use traffic class 0; start traffic class 0's
3124          * queue allocation at queue 0, and assign it 64 (2^6) queues (though
3125          * the driver may not use all of them).
3126          */
3127         ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
3128             & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
3129             ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
3130             & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
3131
3132         /* Set VLAN receive stripping mode */
3133         ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
3134         ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
3135         if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
3136                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3137         else
3138                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3139
3140         /* Keep copy of VSI info in VSI for statistic counters */
3141         memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
3142
3143         /* Reset VSI statistics */
3144         ixl_vsi_reset_stats(vsi);
3145         vsi->hw_filters_add = 0;
3146         vsi->hw_filters_del = 0;
3147
3148         ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
3149
3150         err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3151         if (err) {
3152                 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d, aq_error %d\n",
3153                    err, hw->aq.asq_last_status);
3154                 return (err);
3155         }
3156
3157         for (int i = 0; i < vsi->num_queues; i++, que++) {
3158                 struct tx_ring          *txr = &que->txr;
3159                 struct rx_ring          *rxr = &que->rxr;
3160                 struct i40e_hmc_obj_txq tctx;
3161                 struct i40e_hmc_obj_rxq rctx;
3162                 u32                     txctl;
3163                 u16                     size;
3164
3165                 /* Setup the HMC TX Context  */
3166                 size = que->num_desc * sizeof(struct i40e_tx_desc);
3167                 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
3168                 tctx.new_context = 1;
3169                 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
3170                 tctx.qlen = que->num_desc;
3171                 tctx.fc_ena = 0;
3172                 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
3173                 /* Enable HEAD writeback */
3174                 tctx.head_wb_ena = 1;
3175                 tctx.head_wb_addr = txr->dma.pa +
3176                     (que->num_desc * sizeof(struct i40e_tx_desc));
3177                 tctx.rdylist_act = 0;
3178                 err = i40e_clear_lan_tx_queue_context(hw, i);
3179                 if (err) {
3180                         device_printf(dev, "Unable to clear TX context\n");
3181                         break;
3182                 }
3183                 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
3184                 if (err) {
3185                         device_printf(dev, "Unable to set TX context\n");
3186                         break;
3187                 }
3188                 /* Associate the ring with this PF */
3189                 txctl = I40E_QTX_CTL_PF_QUEUE;
3190                 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3191                     I40E_QTX_CTL_PF_INDX_MASK);
3192                 wr32(hw, I40E_QTX_CTL(i), txctl);
3193                 ixl_flush(hw);
3194
3195                 /* Do ring (re)init */
3196                 ixl_init_tx_ring(que);
3197
3198                 /* Next setup the HMC RX Context  */
3199                 if (vsi->max_frame_size <= MCLBYTES)
3200                         rxr->mbuf_sz = MCLBYTES;
3201                 else
3202                         rxr->mbuf_sz = MJUMPAGESIZE;
3203
3204                 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
3205
3206                 /* Set up an RX context for the HMC */
3207                 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
3208                 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
3209                 /* ignore header split for now */
3210                 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
3211                 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
3212                     vsi->max_frame_size : max_rxmax;
3213                 rctx.dtype = 0;
3214                 rctx.dsize = 1; /* do 32byte descriptors */
3215                 rctx.hsplit_0 = 0;  /* no HDR split initially */
3216                 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
3217                 rctx.qlen = que->num_desc;
3218                 rctx.tphrdesc_ena = 1;
3219                 rctx.tphwdesc_ena = 1;
3220                 rctx.tphdata_ena = 0;
3221                 rctx.tphhead_ena = 0;
3222                 rctx.lrxqthresh = 2;
3223                 rctx.crcstrip = 1;
3224                 rctx.l2tsel = 1;
3225                 rctx.showiv = 1;
3226                 rctx.fc_ena = 0;
3227                 rctx.prefena = 1;
3228
3229                 err = i40e_clear_lan_rx_queue_context(hw, i);
3230                 if (err) {
3231                         device_printf(dev,
3232                             "Unable to clear RX context %d\n", i);
3233                         break;
3234                 }
3235                 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
3236                 if (err) {
3237                         device_printf(dev, "Unable to set RX context %d\n", i);
3238                         break;
3239                 }
3240                 err = ixl_init_rx_ring(que);
3241                 if (err) {
3242                         device_printf(dev, "Fail in init_rx_ring %d\n", i);
3243                         break;
3244                 }
3245 #ifdef DEV_NETMAP
3246                 /* preserve queue */
3247                 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
3248                         struct netmap_adapter *na = NA(vsi->ifp);
3249                         struct netmap_kring *kring = &na->rx_rings[i];
3250                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
3251                         wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
3252                 } else
3253 #endif /* DEV_NETMAP */
3254                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
3255         }
3256         return (err);
3257 }
3258
3259
3260 /*********************************************************************
3261  *
3262  *  Free all VSI structs.
3263  *
3264  **********************************************************************/
3265 void
3266 ixl_free_vsi(struct ixl_vsi *vsi)
3267 {
3268         struct ixl_pf           *pf = (struct ixl_pf *)vsi->back;
3269         struct ixl_queue        *que = vsi->queues;
3270
3271         /* Free station queues */
3272         if (!vsi->queues)
3273                 goto free_filters;
3274
3275         for (int i = 0; i < vsi->num_queues; i++, que++) {
3276                 struct tx_ring *txr = &que->txr;
3277                 struct rx_ring *rxr = &que->rxr;
3278
3279                 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
3280                         continue;
3281                 IXL_TX_LOCK(txr);
3282                 ixl_free_que_tx(que);
3283                 if (txr->base)
3284                         i40e_free_dma_mem(&pf->hw, &txr->dma);
3285                 IXL_TX_UNLOCK(txr);
3286                 IXL_TX_LOCK_DESTROY(txr);
3287
3288                 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
3289                         continue;
3290                 IXL_RX_LOCK(rxr);
3291                 ixl_free_que_rx(que);
3292                 if (rxr->base)
3293                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
3294                 IXL_RX_UNLOCK(rxr);
3295                 IXL_RX_LOCK_DESTROY(rxr);
3296
3297         }
3298         free(vsi->queues, M_DEVBUF);
3299
3300 free_filters:
3301         /* Free VSI filter list */
3302         ixl_free_mac_filters(vsi);
3303 }
3304
3305 static void
3306 ixl_free_mac_filters(struct ixl_vsi *vsi)
3307 {
3308         struct ixl_mac_filter *f;
3309
3310         while (!SLIST_EMPTY(&vsi->ftl)) {
3311                 f = SLIST_FIRST(&vsi->ftl);
3312                 SLIST_REMOVE_HEAD(&vsi->ftl, next);
3313                 free(f, M_DEVBUF);
3314         }
3315 }
3316
3317
3318 /*********************************************************************
3319  *
3320  *  Allocate memory for the VSI (virtual station interface) and their
3321  *  associated queues, rings and the descriptors associated with each,
3322  *  called only once at attach.
3323  *
3324  **********************************************************************/
3325 static int
3326 ixl_setup_stations(struct ixl_pf *pf)
3327 {
3328         device_t                dev = pf->dev;
3329         struct ixl_vsi          *vsi;
3330         struct ixl_queue        *que;
3331         struct tx_ring          *txr;
3332         struct rx_ring          *rxr;
3333         int                     rsize, tsize;
3334         int                     error = I40E_SUCCESS;
3335
3336         vsi = &pf->vsi;
3337         vsi->back = (void *)pf;
3338         vsi->hw = &pf->hw;
3339         vsi->id = 0;
3340         vsi->num_vlans = 0;
3341         vsi->back = pf;
3342
3343         /* Get memory for the station queues */
3344         if (!(vsi->queues =
3345             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
3346             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3347                 device_printf(dev, "Unable to allocate queue memory\n");
3348                 error = ENOMEM;
3349                 goto early;
3350         }
3351
3352         for (int i = 0; i < vsi->num_queues; i++) {
3353                 que = &vsi->queues[i];
3354                 que->num_desc = ixl_ringsz;
3355                 que->me = i;
3356                 que->vsi = vsi;
3357                 /* mark the queue as active */
3358                 vsi->active_queues |= (u64)1 << que->me;
3359                 txr = &que->txr;
3360                 txr->que = que;
3361                 txr->tail = I40E_QTX_TAIL(que->me);
3362
3363                 /* Initialize the TX lock */
3364                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3365                     device_get_nameunit(dev), que->me);
3366                 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
3367                 /* Create the TX descriptor ring */
3368                 tsize = roundup2((que->num_desc *
3369                     sizeof(struct i40e_tx_desc)) +
3370                     sizeof(u32), DBA_ALIGN);
3371                 if (i40e_allocate_dma_mem(&pf->hw,
3372                     &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
3373                         device_printf(dev,
3374                             "Unable to allocate TX Descriptor memory\n");
3375                         error = ENOMEM;
3376                         goto fail;
3377                 }
3378                 txr->base = (struct i40e_tx_desc *)txr->dma.va;
3379                 bzero((void *)txr->base, tsize);
3380                 /* Now allocate transmit soft structs for the ring */
3381                 if (ixl_allocate_tx_data(que)) {
3382                         device_printf(dev,
3383                             "Critical Failure setting up TX structures\n");
3384                         error = ENOMEM;
3385                         goto fail;
3386                 }
3387                 /* Allocate a buf ring */
3388                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3389                     M_NOWAIT, &txr->mtx);
3390                 if (txr->br == NULL) {
3391                         device_printf(dev,
3392                             "Critical Failure setting up TX buf ring\n");
3393                         error = ENOMEM;
3394                         goto fail;
3395                 }
3396
3397                 /*
3398                  * Next the RX queues...
3399                  */
3400                 rsize = roundup2(que->num_desc *
3401                     sizeof(union i40e_rx_desc), DBA_ALIGN);
3402                 rxr = &que->rxr;
3403                 rxr->que = que;
3404                 rxr->tail = I40E_QRX_TAIL(que->me);
3405
3406                 /* Initialize the RX side lock */
3407                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3408                     device_get_nameunit(dev), que->me);
3409                 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3410
3411                 if (i40e_allocate_dma_mem(&pf->hw,
3412                     &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3413                         device_printf(dev,
3414                             "Unable to allocate RX Descriptor memory\n");
3415                         error = ENOMEM;
3416                         goto fail;
3417                 }
3418                 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3419                 bzero((void *)rxr->base, rsize);
3420
3421                 /* Allocate receive soft structs for the ring*/
3422                 if (ixl_allocate_rx_data(que)) {
3423                         device_printf(dev,
3424                             "Critical Failure setting up receive structs\n");
3425                         error = ENOMEM;
3426                         goto fail;
3427                 }
3428         }
3429
3430         return (0);
3431
3432 fail:
3433         for (int i = 0; i < vsi->num_queues; i++) {
3434                 que = &vsi->queues[i];
3435                 rxr = &que->rxr;
3436                 txr = &que->txr;
3437                 if (rxr->base)
3438                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
3439                 if (txr->base)
3440                         i40e_free_dma_mem(&pf->hw, &txr->dma);
3441         }
3442
3443 early:
3444         return (error);
3445 }
3446
3447 /*
3448 ** Provide a update to the queue RX
3449 ** interrupt moderation value.
3450 */
3451 static void
3452 ixl_set_queue_rx_itr(struct ixl_queue *que)
3453 {
3454         struct ixl_vsi  *vsi = que->vsi;
3455         struct i40e_hw  *hw = vsi->hw;
3456         struct rx_ring  *rxr = &que->rxr;
3457         u16             rx_itr;
3458         u16             rx_latency = 0;
3459         int             rx_bytes;
3460
3461         /* Idle, do nothing */
3462         if (rxr->bytes == 0)
3463                 return;
3464
3465         if (ixl_dynamic_rx_itr) {
3466                 rx_bytes = rxr->bytes/rxr->itr;
3467                 rx_itr = rxr->itr;
3468
3469                 /* Adjust latency range */
3470                 switch (rxr->latency) {
3471                 case IXL_LOW_LATENCY:
3472                         if (rx_bytes > 10) {
3473                                 rx_latency = IXL_AVE_LATENCY;
3474                                 rx_itr = IXL_ITR_20K;
3475                         }
3476                         break;
3477                 case IXL_AVE_LATENCY:
3478                         if (rx_bytes > 20) {
3479                                 rx_latency = IXL_BULK_LATENCY;
3480                                 rx_itr = IXL_ITR_8K;
3481                         } else if (rx_bytes <= 10) {
3482                                 rx_latency = IXL_LOW_LATENCY;
3483                                 rx_itr = IXL_ITR_100K;
3484                         }
3485                         break;
3486                 case IXL_BULK_LATENCY:
3487                         if (rx_bytes <= 20) {
3488                                 rx_latency = IXL_AVE_LATENCY;
3489                                 rx_itr = IXL_ITR_20K;
3490                         }
3491                         break;
3492                  }
3493
3494                 rxr->latency = rx_latency;
3495
3496                 if (rx_itr != rxr->itr) {
3497                         /* do an exponential smoothing */
3498                         rx_itr = (10 * rx_itr * rxr->itr) /
3499                             ((9 * rx_itr) + rxr->itr);
3500                         rxr->itr = rx_itr & IXL_MAX_ITR;
3501                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3502                             que->me), rxr->itr);
3503                 }
3504         } else { /* We may have have toggled to non-dynamic */
3505                 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3506                         vsi->rx_itr_setting = ixl_rx_itr;
3507                 /* Update the hardware if needed */
3508                 if (rxr->itr != vsi->rx_itr_setting) {
3509                         rxr->itr = vsi->rx_itr_setting;
3510                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3511                             que->me), rxr->itr);
3512                 }
3513         }
3514         rxr->bytes = 0;
3515         rxr->packets = 0;
3516         return;
3517 }
3518
3519
3520 /*
3521 ** Provide a update to the queue TX
3522 ** interrupt moderation value.
3523 */
3524 static void
3525 ixl_set_queue_tx_itr(struct ixl_queue *que)
3526 {
3527         struct ixl_vsi  *vsi = que->vsi;
3528         struct i40e_hw  *hw = vsi->hw;
3529         struct tx_ring  *txr = &que->txr;
3530         u16             tx_itr;
3531         u16             tx_latency = 0;
3532         int             tx_bytes;
3533
3534
3535         /* Idle, do nothing */
3536         if (txr->bytes == 0)
3537                 return;
3538
3539         if (ixl_dynamic_tx_itr) {
3540                 tx_bytes = txr->bytes/txr->itr;
3541                 tx_itr = txr->itr;
3542
3543                 switch (txr->latency) {
3544                 case IXL_LOW_LATENCY:
3545                         if (tx_bytes > 10) {
3546                                 tx_latency = IXL_AVE_LATENCY;
3547                                 tx_itr = IXL_ITR_20K;
3548                         }
3549                         break;
3550                 case IXL_AVE_LATENCY:
3551                         if (tx_bytes > 20) {
3552                                 tx_latency = IXL_BULK_LATENCY;
3553                                 tx_itr = IXL_ITR_8K;
3554                         } else if (tx_bytes <= 10) {
3555                                 tx_latency = IXL_LOW_LATENCY;
3556                                 tx_itr = IXL_ITR_100K;
3557                         }
3558                         break;
3559                 case IXL_BULK_LATENCY:
3560                         if (tx_bytes <= 20) {
3561                                 tx_latency = IXL_AVE_LATENCY;
3562                                 tx_itr = IXL_ITR_20K;
3563                         }
3564                         break;
3565                 }
3566
3567                 txr->latency = tx_latency;
3568
3569                 if (tx_itr != txr->itr) {
3570                  /* do an exponential smoothing */
3571                         tx_itr = (10 * tx_itr * txr->itr) /
3572                             ((9 * tx_itr) + txr->itr);
3573                         txr->itr = tx_itr & IXL_MAX_ITR;
3574                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3575                             que->me), txr->itr);
3576                 }
3577
3578         } else { /* We may have have toggled to non-dynamic */
3579                 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3580                         vsi->tx_itr_setting = ixl_tx_itr;
3581                 /* Update the hardware if needed */
3582                 if (txr->itr != vsi->tx_itr_setting) {
3583                         txr->itr = vsi->tx_itr_setting;
3584                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3585                             que->me), txr->itr);
3586                 }
3587         }
3588         txr->bytes = 0;
3589         txr->packets = 0;
3590         return;
3591 }
3592
3593 #define QUEUE_NAME_LEN 32
3594
3595 static void
3596 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3597     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3598 {
3599         struct sysctl_oid *tree;
3600         struct sysctl_oid_list *child;
3601         struct sysctl_oid_list *vsi_list;
3602
3603         tree = device_get_sysctl_tree(pf->dev);
3604         child = SYSCTL_CHILDREN(tree);
3605         vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3606                                    CTLFLAG_RD, NULL, "VSI Number");
3607         vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3608
3609         ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3610 }
3611
3612 #ifdef IXL_DEBUG
3613 /**
3614  * ixl_sysctl_qtx_tail_handler
3615  * Retrieves I40E_QTX_TAIL value from hardware
3616  * for a sysctl.
3617  */
3618 static int
3619 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)
3620 {
3621         struct ixl_queue *que;
3622         int error;
3623         u32 val;
3624
3625         que = ((struct ixl_queue *)oidp->oid_arg1);
3626         if (!que) return 0;
3627
3628         val = rd32(que->vsi->hw, que->txr.tail);
3629         error = sysctl_handle_int(oidp, &val, 0, req);
3630         if (error || !req->newptr)
3631                 return error;
3632         return (0);
3633 }
3634
3635 /**
3636  * ixl_sysctl_qrx_tail_handler
3637  * Retrieves I40E_QRX_TAIL value from hardware
3638  * for a sysctl.
3639  */
3640 static int
3641 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)
3642 {
3643         struct ixl_queue *que;
3644         int error;
3645         u32 val;
3646
3647         que = ((struct ixl_queue *)oidp->oid_arg1);
3648         if (!que) return 0;
3649
3650         val = rd32(que->vsi->hw, que->rxr.tail);
3651         error = sysctl_handle_int(oidp, &val, 0, req);
3652         if (error || !req->newptr)
3653                 return error;
3654         return (0);
3655 }
3656 #endif
3657
3658 static void
3659 ixl_add_hw_stats(struct ixl_pf *pf)
3660 {
3661         device_t dev = pf->dev;
3662         struct ixl_vsi *vsi = &pf->vsi;
3663         struct ixl_queue *queues = vsi->queues;
3664         struct i40e_hw_port_stats *pf_stats = &pf->stats;
3665
3666         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3667         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3668         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3669         struct sysctl_oid_list *vsi_list;
3670
3671         struct sysctl_oid *queue_node;
3672         struct sysctl_oid_list *queue_list;
3673
3674         struct tx_ring *txr;
3675         struct rx_ring *rxr;
3676         char queue_namebuf[QUEUE_NAME_LEN];
3677
3678         /* Driver statistics */
3679         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3680                         CTLFLAG_RD, &pf->watchdog_events,
3681                         "Watchdog timeouts");
3682         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3683                         CTLFLAG_RD, &pf->admin_irq,
3684                         "Admin Queue IRQ Handled");
3685
3686         ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3687         vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3688
3689         /* Queue statistics */
3690         for (int q = 0; q < vsi->num_queues; q++) {
3691                 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3692                 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3693                     OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3694                 queue_list = SYSCTL_CHILDREN(queue_node);
3695
3696                 txr = &(queues[q].txr);
3697                 rxr = &(queues[q].rxr);
3698
3699                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3700                                 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3701                                 "m_defrag() failed");
3702                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3703                                 CTLFLAG_RD, &(queues[q].irqs),
3704                                 "irqs on this queue");
3705                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3706                                 CTLFLAG_RD, &(queues[q].tso),
3707                                 "TSO");
3708                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3709                                 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3710                                 "Driver tx dma failure in xmit");
3711                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3712                                 CTLFLAG_RD, &(txr->no_desc),
3713                                 "Queue No Descriptor Available");
3714                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3715                                 CTLFLAG_RD, &(txr->total_packets),
3716                                 "Queue Packets Transmitted");
3717                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3718                                 CTLFLAG_RD, &(txr->tx_bytes),
3719                                 "Queue Bytes Transmitted");
3720                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3721                                 CTLFLAG_RD, &(rxr->rx_packets),
3722                                 "Queue Packets Received");
3723                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3724                                 CTLFLAG_RD, &(rxr->rx_bytes),
3725                                 "Queue Bytes Received");
3726                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err",
3727                                 CTLFLAG_RD, &(rxr->desc_errs),
3728                                 "Queue Rx Descriptor Errors");
3729                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr",
3730                                 CTLFLAG_RD, &(rxr->itr), 0,
3731                                 "Queue Rx ITR Interval");
3732                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr",
3733                                 CTLFLAG_RD, &(txr->itr), 0,
3734                                 "Queue Tx ITR Interval");
3735                 // Not actual latency; just a calculated value to put in a register
3736                 // TODO: Put in better descriptions here
3737                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_latency",
3738                                 CTLFLAG_RD, &(rxr->latency), 0,
3739                                 "Queue Rx ITRL Average Interval");
3740                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_latency",
3741                                 CTLFLAG_RD, &(txr->latency), 0,
3742                                 "Queue Tx ITRL Average Interval");
3743
3744 #ifdef IXL_DEBUG
3745                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done",
3746                                 CTLFLAG_RD, &(rxr->not_done),
3747                                 "Queue Rx Descriptors not Done");
3748                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh",
3749                                 CTLFLAG_RD, &(rxr->next_refresh), 0,
3750                                 "Queue Rx Descriptors not Done");
3751                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check",
3752                                 CTLFLAG_RD, &(rxr->next_check), 0,
3753                                 "Queue Rx Descriptors not Done");
3754                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail",
3755                                 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3756                                 sizeof(struct ixl_queue),
3757                                 ixl_sysctl_qtx_tail_handler, "IU",
3758                                 "Queue Transmit Descriptor Tail");
3759                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail",
3760                                 CTLTYPE_UINT | CTLFLAG_RD, &queues[q],
3761                                 sizeof(struct ixl_queue),
3762                                 ixl_sysctl_qrx_tail_handler, "IU",
3763                                 "Queue Receive Descriptor Tail");
3764 #endif
3765         }
3766
3767         /* MAC stats */
3768         ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3769 }
3770
3771 static void
3772 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3773         struct sysctl_oid_list *child,
3774         struct i40e_eth_stats *eth_stats)
3775 {
3776         struct ixl_sysctl_info ctls[] =
3777         {
3778                 {&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3779                 {&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3780                         "Unicast Packets Received"},
3781                 {&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3782                         "Multicast Packets Received"},
3783                 {&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3784                         "Broadcast Packets Received"},
3785                 {&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3786                 {&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3787                 {&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3788                 {&eth_stats->tx_multicast, "mcast_pkts_txd",
3789                         "Multicast Packets Transmitted"},
3790                 {&eth_stats->tx_broadcast, "bcast_pkts_txd",
3791                         "Broadcast Packets Transmitted"},
3792                 // end
3793                 {0,0,0}
3794         };
3795
3796         struct ixl_sysctl_info *entry = ctls;
3797         while (entry->stat != NULL)
3798         {
3799                 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3800                                 CTLFLAG_RD, entry->stat,
3801                                 entry->description);
3802                 entry++;
3803         }
3804 }
3805
3806 static void
3807 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3808         struct sysctl_oid_list *child,
3809         struct i40e_hw_port_stats *stats)
3810 {
3811         struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3812                                     CTLFLAG_RD, NULL, "Mac Statistics");
3813         struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3814
3815         struct i40e_eth_stats *eth_stats = &stats->eth;
3816         ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3817
3818         struct ixl_sysctl_info ctls[] =
3819         {
3820                 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3821                 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3822                 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3823                 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3824                 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3825                 /* Packet Reception Stats */
3826                 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3827                 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3828                 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3829                 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3830                 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3831                 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3832                 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3833                 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3834                 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3835                 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3836                 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3837                 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3838                 /* Packet Transmission Stats */
3839                 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3840                 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3841                 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3842                 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3843                 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3844                 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3845                 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3846                 /* Flow control */
3847                 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3848                 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3849                 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3850                 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3851                 /* End */
3852                 {0,0,0}
3853         };
3854
3855         struct ixl_sysctl_info *entry = ctls;
3856         while (entry->stat != NULL)
3857         {
3858                 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3859                                 CTLFLAG_RD, entry->stat,
3860                                 entry->description);
3861                 entry++;
3862         }
3863 }
3864
3865
3866 /*
3867 ** ixl_config_rss - setup RSS
3868 **  - note this is done for the single vsi
3869 */
3870 static void
3871 ixl_config_rss(struct ixl_vsi *vsi)
3872 {
3873         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3874         struct i40e_hw  *hw = vsi->hw;
3875         u32             lut = 0;
3876         u64             set_hena = 0, hena;
3877         int             i, j, que_id;
3878 #ifdef RSS
3879         u32             rss_hash_config;
3880         u32             rss_seed[IXL_KEYSZ];
3881 #else
3882         u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3883                             0x183cfd8c, 0xce880440, 0x580cbc3c,
3884                             0x35897377, 0x328b25e1, 0x4fa98922,
3885                             0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3886 #endif
3887
3888 #ifdef RSS
3889         /* Fetch the configured RSS key */
3890         rss_getkey((uint8_t *) &rss_seed);
3891 #endif
3892
3893         /* Fill out hash function seed */
3894         for (i = 0; i < IXL_KEYSZ; i++)
3895                 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3896
3897         /* Enable PCTYPES for RSS: */
3898 #ifdef RSS
3899         rss_hash_config = rss_gethashconfig();
3900         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3901                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3902         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3903                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3904         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3905                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3906         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3907                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3908         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3909                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3910         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3911                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3912         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3913                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3914 #else
3915         set_hena =
3916                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3917                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3918                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3919                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3920                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3921                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3922                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3923                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3924                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3925                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3926                 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3927 #endif
3928         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
3929             ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
3930         hena |= set_hena;
3931         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
3932         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3933
3934         /* Populate the LUT with max no. of queues in round robin fashion */
3935         for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3936                 if (j == vsi->num_queues)
3937                         j = 0;
3938 #ifdef RSS
3939                 /*
3940                  * Fetch the RSS bucket id for the given indirection entry.
3941                  * Cap it at the number of configured buckets (which is
3942                  * num_queues.)
3943                  */
3944                 que_id = rss_get_indirection_to_bucket(i);
3945                 que_id = que_id % vsi->num_queues;
3946 #else
3947                 que_id = j;
3948 #endif
3949                 /* lut = 4-byte sliding window of 4 lut entries */
3950                 lut = (lut << 8) | (que_id &
3951                     ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3952                 /* On i = 3, we have 4 entries in lut; write to the register */
3953                 if ((i & 3) == 3)
3954                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3955         }
3956         ixl_flush(hw);
3957 }
3958
3959
3960 /*
3961 ** This routine is run via an vlan config EVENT,
3962 ** it enables us to use the HW Filter table since
3963 ** we can get the vlan id. This just creates the
3964 ** entry in the soft version of the VFTA, init will
3965 ** repopulate the real table.
3966 */
3967 static void
3968 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3969 {
3970         struct ixl_vsi  *vsi = ifp->if_softc;
3971         struct i40e_hw  *hw = vsi->hw;
3972         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3973
3974         if (ifp->if_softc !=  arg)   /* Not our event */
3975                 return;
3976
3977         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3978                 return;
3979
3980         IXL_PF_LOCK(pf);
3981         ++vsi->num_vlans;
3982         ixl_add_filter(vsi, hw->mac.addr, vtag);
3983         IXL_PF_UNLOCK(pf);
3984 }
3985
3986 /*
3987 ** This routine is run via an vlan
3988 ** unconfig EVENT, remove our entry
3989 ** in the soft vfta.
3990 */
3991 static void
3992 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3993 {
3994         struct ixl_vsi  *vsi = ifp->if_softc;
3995         struct i40e_hw  *hw = vsi->hw;
3996         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3997
3998         if (ifp->if_softc !=  arg)
3999                 return;
4000
4001         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4002                 return;
4003
4004         IXL_PF_LOCK(pf);
4005         --vsi->num_vlans;
4006         ixl_del_filter(vsi, hw->mac.addr, vtag);
4007         IXL_PF_UNLOCK(pf);
4008 }
4009
4010 /*
4011 ** This routine updates vlan filters, called by init
4012 ** it scans the filter table and then updates the hw
4013 ** after a soft reset.
4014 */
4015 static void
4016 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
4017 {
4018         struct ixl_mac_filter   *f;
4019         int                     cnt = 0, flags;
4020
4021         if (vsi->num_vlans == 0)
4022                 return;
4023         /*
4024         ** Scan the filter list for vlan entries,
4025         ** mark them for addition and then call
4026         ** for the AQ update.
4027         */
4028         SLIST_FOREACH(f, &vsi->ftl, next) {
4029                 if (f->flags & IXL_FILTER_VLAN) {
4030                         f->flags |=
4031                             (IXL_FILTER_ADD |
4032                             IXL_FILTER_USED);
4033                         cnt++;
4034                 }
4035         }
4036         if (cnt == 0) {
4037                 printf("setup vlan: no filters found!\n");
4038                 return;
4039         }
4040         flags = IXL_FILTER_VLAN;
4041         flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4042         ixl_add_hw_filters(vsi, flags, cnt);
4043         return;
4044 }
4045
4046 /*
4047 ** Initialize filter list and add filters that the hardware
4048 ** needs to know about.
4049 **
4050 ** Requires VSI's filter list & seid to be set before calling.
4051 */
4052 static void
4053 ixl_init_filters(struct ixl_vsi *vsi)
4054 {
4055         /* Add broadcast address */
4056         ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
4057
4058         /*
4059          * Prevent Tx flow control frames from being sent out by
4060          * non-firmware transmitters.
4061          */
4062         i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid);
4063 }
4064
4065 /*
4066 ** This routine adds mulicast filters
4067 */
4068 static void
4069 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
4070 {
4071         struct ixl_mac_filter *f;
4072
4073         /* Does one already exist */
4074         f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4075         if (f != NULL)
4076                 return;
4077
4078         f = ixl_get_filter(vsi);
4079         if (f == NULL) {
4080                 printf("WARNING: no filter available!!\n");
4081                 return;
4082         }
4083         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4084         f->vlan = IXL_VLAN_ANY;
4085         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
4086             | IXL_FILTER_MC);
4087
4088         return;
4089 }
4090
4091 static void
4092 ixl_reconfigure_filters(struct ixl_vsi *vsi)
4093 {
4094
4095         ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
4096 }
4097
4098 /*
4099 ** This routine adds macvlan filters
4100 */
4101 static void
4102 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4103 {
4104         struct ixl_mac_filter   *f, *tmp;
4105         struct ixl_pf           *pf;
4106         device_t                dev;
4107
4108         DEBUGOUT("ixl_add_filter: begin");
4109
4110         pf = vsi->back;
4111         dev = pf->dev;
4112
4113         /* Does one already exist */
4114         f = ixl_find_filter(vsi, macaddr, vlan);
4115         if (f != NULL)
4116                 return;
4117         /*
4118         ** Is this the first vlan being registered, if so we
4119         ** need to remove the ANY filter that indicates we are
4120         ** not in a vlan, and replace that with a 0 filter.
4121         */
4122         if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
4123                 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
4124                 if (tmp != NULL) {
4125                         ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
4126                         ixl_add_filter(vsi, macaddr, 0);
4127                 }
4128         }
4129
4130         f = ixl_get_filter(vsi);
4131         if (f == NULL) {
4132                 device_printf(dev, "WARNING: no filter available!!\n");
4133                 return;
4134         }
4135         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
4136         f->vlan = vlan;
4137         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
4138         if (f->vlan != IXL_VLAN_ANY)
4139                 f->flags |= IXL_FILTER_VLAN;
4140         else
4141                 vsi->num_macs++;
4142
4143         ixl_add_hw_filters(vsi, f->flags, 1);
4144         return;
4145 }
4146
4147 static void
4148 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4149 {
4150         struct ixl_mac_filter *f;
4151
4152         f = ixl_find_filter(vsi, macaddr, vlan);
4153         if (f == NULL)
4154                 return;
4155
4156         f->flags |= IXL_FILTER_DEL;
4157         ixl_del_hw_filters(vsi, 1);
4158         vsi->num_macs--;
4159
4160         /* Check if this is the last vlan removal */
4161         if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
4162                 /* Switch back to a non-vlan filter */
4163                 ixl_del_filter(vsi, macaddr, 0);
4164                 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
4165         }
4166         return;
4167 }
4168
4169 /*
4170 ** Find the filter with both matching mac addr and vlan id
4171 */
4172 static struct ixl_mac_filter *
4173 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
4174 {
4175         struct ixl_mac_filter   *f;
4176         bool                    match = FALSE;
4177
4178         SLIST_FOREACH(f, &vsi->ftl, next) {
4179                 if (!cmp_etheraddr(f->macaddr, macaddr))
4180                         continue;
4181                 if (f->vlan == vlan) {
4182                         match = TRUE;
4183                         break;
4184                 }
4185         }
4186
4187         if (!match)
4188                 f = NULL;
4189         return (f);
4190 }
4191
4192 /*
4193 ** This routine takes additions to the vsi filter
4194 ** table and creates an Admin Queue call to create
4195 ** the filters in the hardware.
4196 */
4197 static void
4198 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
4199 {
4200         struct i40e_aqc_add_macvlan_element_data *a, *b;
4201         struct ixl_mac_filter   *f;
4202         struct ixl_pf           *pf;
4203         struct i40e_hw          *hw;
4204         device_t                dev;
4205         int                     err, j = 0;
4206
4207         pf = vsi->back;
4208         dev = pf->dev;
4209         hw = &pf->hw;
4210         IXL_PF_LOCK_ASSERT(pf);
4211
4212         a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
4213             M_DEVBUF, M_NOWAIT | M_ZERO);
4214         if (a == NULL) {
4215                 device_printf(dev, "add_hw_filters failed to get memory\n");
4216                 return;
4217         }
4218
4219         /*
4220         ** Scan the filter list, each time we find one
4221         ** we add it to the admin queue array and turn off
4222         ** the add bit.
4223         */
4224         SLIST_FOREACH(f, &vsi->ftl, next) {
4225                 if (f->flags == flags) {
4226                         b = &a[j]; // a pox on fvl long names :)
4227                         bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
4228                         if (f->vlan == IXL_VLAN_ANY) {
4229                                 b->vlan_tag = 0;
4230                                 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
4231                         } else {
4232                                 b->vlan_tag = f->vlan;
4233                                 b->flags = 0;
4234                         }
4235                         b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
4236                         f->flags &= ~IXL_FILTER_ADD;
4237                         j++;
4238                 }
4239                 if (j == cnt)
4240                         break;
4241         }
4242         if (j > 0) {
4243                 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
4244                 if (err)
4245                         device_printf(dev, "aq_add_macvlan err %d, "
4246                             "aq_error %d\n", err, hw->aq.asq_last_status);
4247                 else
4248                         vsi->hw_filters_add += j;
4249         }
4250         free(a, M_DEVBUF);
4251         return;
4252 }
4253
4254 /*
4255 ** This routine takes removals in the vsi filter
4256 ** table and creates an Admin Queue call to delete
4257 ** the filters in the hardware.
4258 */
4259 static void
4260 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
4261 {
4262         struct i40e_aqc_remove_macvlan_element_data *d, *e;
4263         struct ixl_pf           *pf;
4264         struct i40e_hw          *hw;
4265         device_t                dev;
4266         struct ixl_mac_filter   *f, *f_temp;
4267         int                     err, j = 0;
4268
4269         DEBUGOUT("ixl_del_hw_filters: begin\n");
4270
4271         pf = vsi->back;
4272         hw = &pf->hw;
4273         dev = pf->dev;
4274
4275         d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
4276             M_DEVBUF, M_NOWAIT | M_ZERO);
4277         if (d == NULL) {
4278                 printf("del hw filter failed to get memory\n");
4279                 return;
4280         }
4281
4282         SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
4283                 if (f->flags & IXL_FILTER_DEL) {
4284                         e = &d[j]; // a pox on fvl long names :)
4285                         bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
4286                         e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
4287                         e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
4288                         /* delete entry from vsi list */
4289                         SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
4290                         free(f, M_DEVBUF);
4291                         j++;
4292                 }
4293                 if (j == cnt)
4294                         break;
4295         }
4296         if (j > 0) {
4297                 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
4298                 /* NOTE: returns ENOENT every time but seems to work fine,
4299                    so we'll ignore that specific error. */
4300                 // TODO: Does this still occur on current firmwares?
4301                 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
4302                         int sc = 0;
4303                         for (int i = 0; i < j; i++)
4304                                 sc += (!d[i].error_code);
4305                         vsi->hw_filters_del += sc;
4306                         device_printf(dev,
4307                             "Failed to remove %d/%d filters, aq error %d\n",
4308                             j - sc, j, hw->aq.asq_last_status);
4309                 } else
4310                         vsi->hw_filters_del += j;
4311         }
4312         free(d, M_DEVBUF);
4313
4314         DEBUGOUT("ixl_del_hw_filters: end\n");
4315         return;
4316 }
4317
4318 static int
4319 ixl_enable_rings(struct ixl_vsi *vsi)
4320 {
4321         struct ixl_pf   *pf = vsi->back;
4322         struct i40e_hw  *hw = &pf->hw;
4323         int             index, error;
4324         u32             reg;
4325
4326         error = 0;
4327         for (int i = 0; i < vsi->num_queues; i++) {
4328                 index = vsi->first_queue + i;
4329                 i40e_pre_tx_queue_cfg(hw, index, TRUE);
4330
4331                 reg = rd32(hw, I40E_QTX_ENA(index));
4332                 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
4333                     I40E_QTX_ENA_QENA_STAT_MASK;
4334                 wr32(hw, I40E_QTX_ENA(index), reg);
4335                 /* Verify the enable took */
4336                 for (int j = 0; j < 10; j++) {
4337                         reg = rd32(hw, I40E_QTX_ENA(index));
4338                         if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4339                                 break;
4340                         i40e_msec_delay(10);
4341                 }
4342                 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
4343                         device_printf(pf->dev, "TX queue %d disabled!\n",
4344                             index);
4345                         error = ETIMEDOUT;
4346                 }
4347
4348                 reg = rd32(hw, I40E_QRX_ENA(index));
4349                 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
4350                     I40E_QRX_ENA_QENA_STAT_MASK;
4351                 wr32(hw, I40E_QRX_ENA(index), reg);
4352                 /* Verify the enable took */
4353                 for (int j = 0; j < 10; j++) {
4354                         reg = rd32(hw, I40E_QRX_ENA(index));
4355                         if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
4356                                 break;
4357                         i40e_msec_delay(10);
4358                 }
4359                 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
4360                         device_printf(pf->dev, "RX queue %d disabled!\n",
4361                             index);
4362                         error = ETIMEDOUT;
4363                 }
4364         }
4365
4366         return (error);
4367 }
4368
4369 static int
4370 ixl_disable_rings(struct ixl_vsi *vsi)
4371 {
4372         struct ixl_pf   *pf = vsi->back;
4373         struct i40e_hw  *hw = &pf->hw;
4374         int             index, error;
4375         u32             reg;
4376
4377         error = 0;
4378         for (int i = 0; i < vsi->num_queues; i++) {
4379                 index = vsi->first_queue + i;
4380
4381                 i40e_pre_tx_queue_cfg(hw, index, FALSE);
4382                 i40e_usec_delay(500);
4383
4384                 reg = rd32(hw, I40E_QTX_ENA(index));
4385                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4386                 wr32(hw, I40E_QTX_ENA(index), reg);
4387                 /* Verify the disable took */
4388                 for (int j = 0; j < 10; j++) {
4389                         reg = rd32(hw, I40E_QTX_ENA(index));
4390                         if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4391                                 break;
4392                         i40e_msec_delay(10);
4393                 }
4394                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
4395                         device_printf(pf->dev, "TX queue %d still enabled!\n",
4396                             index);
4397                         error = ETIMEDOUT;
4398                 }
4399
4400                 reg = rd32(hw, I40E_QRX_ENA(index));
4401                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4402                 wr32(hw, I40E_QRX_ENA(index), reg);
4403                 /* Verify the disable took */
4404                 for (int j = 0; j < 10; j++) {
4405                         reg = rd32(hw, I40E_QRX_ENA(index));
4406                         if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
4407                                 break;
4408                         i40e_msec_delay(10);
4409                 }
4410                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
4411                         device_printf(pf->dev, "RX queue %d still enabled!\n",
4412                             index);
4413                         error = ETIMEDOUT;
4414                 }
4415         }
4416
4417         return (error);
4418 }
4419
4420 /**
4421  * ixl_handle_mdd_event
4422  *
4423  * Called from interrupt handler to identify possibly malicious vfs
4424  * (But also detects events from the PF, as well)
4425  **/
4426 static void
4427 ixl_handle_mdd_event(struct ixl_pf *pf)
4428 {
4429         struct i40e_hw *hw = &pf->hw;
4430         device_t dev = pf->dev;
4431         bool mdd_detected = false;
4432         bool pf_mdd_detected = false;
4433         u32 reg;
4434
4435         /* find what triggered the MDD event */
4436         reg = rd32(hw, I40E_GL_MDET_TX);
4437         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
4438                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
4439                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
4440                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
4441                                 I40E_GL_MDET_TX_EVENT_SHIFT;
4442                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
4443                                 I40E_GL_MDET_TX_QUEUE_SHIFT;
4444                 device_printf(dev,
4445                          "Malicious Driver Detection event 0x%02x"
4446                          " on TX queue %d pf number 0x%02x\n",
4447                          event, queue, pf_num);
4448                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
4449                 mdd_detected = true;
4450         }
4451         reg = rd32(hw, I40E_GL_MDET_RX);
4452         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4453                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
4454                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
4455                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
4456                                 I40E_GL_MDET_RX_EVENT_SHIFT;
4457                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
4458                                 I40E_GL_MDET_RX_QUEUE_SHIFT;
4459                 device_printf(dev,
4460                          "Malicious Driver Detection event 0x%02x"
4461                          " on RX queue %d of function 0x%02x\n",
4462                          event, queue, func);
4463                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4464                 mdd_detected = true;
4465         }
4466
4467         if (mdd_detected) {
4468                 reg = rd32(hw, I40E_PF_MDET_TX);
4469                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4470                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4471                         device_printf(dev,
4472                                  "MDD TX event is for this function 0x%08x",
4473                                  reg);
4474                         pf_mdd_detected = true;
4475                 }
4476                 reg = rd32(hw, I40E_PF_MDET_RX);
4477                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4478                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4479                         device_printf(dev,
4480                                  "MDD RX event is for this function 0x%08x",
4481                                  reg);
4482                         pf_mdd_detected = true;
4483                 }
4484         }
4485
4486         /* re-enable mdd interrupt cause */
4487         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4488         reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4489         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4490         ixl_flush(hw);
4491 }
4492
4493 static void
4494 ixl_enable_intr(struct ixl_vsi *vsi)
4495 {
4496         struct i40e_hw          *hw = vsi->hw;
4497         struct ixl_queue        *que = vsi->queues;
4498
4499         if (ixl_enable_msix) {
4500                 for (int i = 0; i < vsi->num_queues; i++, que++)
4501                         ixl_enable_queue(hw, que->me);
4502         } else
4503                 ixl_enable_legacy(hw);
4504 }
4505
4506 static void
4507 ixl_disable_rings_intr(struct ixl_vsi *vsi)
4508 {
4509         struct i40e_hw          *hw = vsi->hw;
4510         struct ixl_queue        *que = vsi->queues;
4511
4512         for (int i = 0; i < vsi->num_queues; i++, que++)
4513                 ixl_disable_queue(hw, que->me);
4514 }
4515
4516 static void
4517 ixl_disable_intr(struct ixl_vsi *vsi)
4518 {
4519         struct i40e_hw          *hw = vsi->hw;
4520
4521         if (ixl_enable_msix)
4522                 ixl_disable_adminq(hw);
4523         else
4524                 ixl_disable_legacy(hw);
4525 }
4526
4527 static void
4528 ixl_enable_adminq(struct i40e_hw *hw)
4529 {
4530         u32             reg;
4531
4532         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4533             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4534             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4535         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4536         ixl_flush(hw);
4537 }
4538
4539 static void
4540 ixl_disable_adminq(struct i40e_hw *hw)
4541 {
4542         u32             reg;
4543
4544         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4545         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4546         ixl_flush(hw);
4547 }
4548
4549 static void
4550 ixl_enable_queue(struct i40e_hw *hw, int id)
4551 {
4552         u32             reg;
4553
4554         reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4555             I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4556             (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4557         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4558 }
4559
4560 static void
4561 ixl_disable_queue(struct i40e_hw *hw, int id)
4562 {
4563         u32             reg;
4564
4565         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4566         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4567 }
4568
4569 static void
4570 ixl_enable_legacy(struct i40e_hw *hw)
4571 {
4572         u32             reg;
4573         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4574             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4575             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4576         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4577 }
4578
4579 static void
4580 ixl_disable_legacy(struct i40e_hw *hw)
4581 {
4582         u32             reg;
4583
4584         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4585         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4586 }
4587
4588 static void
4589 ixl_update_stats_counters(struct ixl_pf *pf)
4590 {
4591         struct i40e_hw  *hw = &pf->hw;
4592         struct ixl_vsi  *vsi = &pf->vsi;
4593         struct ixl_vf   *vf;
4594
4595         struct i40e_hw_port_stats *nsd = &pf->stats;
4596         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4597
4598         /* Update hw stats */
4599         ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4600                            pf->stat_offsets_loaded,
4601                            &osd->crc_errors, &nsd->crc_errors);
4602         ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4603                            pf->stat_offsets_loaded,
4604                            &osd->illegal_bytes, &nsd->illegal_bytes);
4605         ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4606                            I40E_GLPRT_GORCL(hw->port),
4607                            pf->stat_offsets_loaded,
4608                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4609         ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4610                            I40E_GLPRT_GOTCL(hw->port),
4611                            pf->stat_offsets_loaded,
4612                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4613         ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4614                            pf->stat_offsets_loaded,
4615                            &osd->eth.rx_discards,
4616                            &nsd->eth.rx_discards);
4617         ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4618                            I40E_GLPRT_UPRCL(hw->port),
4619                            pf->stat_offsets_loaded,
4620                            &osd->eth.rx_unicast,
4621                            &nsd->eth.rx_unicast);
4622         ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4623                            I40E_GLPRT_UPTCL(hw->port),
4624                            pf->stat_offsets_loaded,
4625                            &osd->eth.tx_unicast,
4626                            &nsd->eth.tx_unicast);
4627         ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4628                            I40E_GLPRT_MPRCL(hw->port),
4629                            pf->stat_offsets_loaded,
4630                            &osd->eth.rx_multicast,
4631                            &nsd->eth.rx_multicast);
4632         ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4633                            I40E_GLPRT_MPTCL(hw->port),
4634                            pf->stat_offsets_loaded,
4635                            &osd->eth.tx_multicast,
4636                            &nsd->eth.tx_multicast);
4637         ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4638                            I40E_GLPRT_BPRCL(hw->port),
4639                            pf->stat_offsets_loaded,
4640                            &osd->eth.rx_broadcast,
4641                            &nsd->eth.rx_broadcast);
4642         ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4643                            I40E_GLPRT_BPTCL(hw->port),
4644                            pf->stat_offsets_loaded,
4645                            &osd->eth.tx_broadcast,
4646                            &nsd->eth.tx_broadcast);
4647
4648         ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4649                            pf->stat_offsets_loaded,
4650                            &osd->tx_dropped_link_down,
4651                            &nsd->tx_dropped_link_down);
4652         ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4653                            pf->stat_offsets_loaded,
4654                            &osd->mac_local_faults,
4655                            &nsd->mac_local_faults);
4656         ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4657                            pf->stat_offsets_loaded,
4658                            &osd->mac_remote_faults,
4659                            &nsd->mac_remote_faults);
4660         ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4661                            pf->stat_offsets_loaded,
4662                            &osd->rx_length_errors,
4663                            &nsd->rx_length_errors);
4664
4665         /* Flow control (LFC) stats */
4666         ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4667                            pf->stat_offsets_loaded,
4668                            &osd->link_xon_rx, &nsd->link_xon_rx);
4669         ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4670                            pf->stat_offsets_loaded,
4671                            &osd->link_xon_tx, &nsd->link_xon_tx);
4672         ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4673                            pf->stat_offsets_loaded,
4674                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
4675         ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4676                            pf->stat_offsets_loaded,
4677                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
4678
4679         /* Packet size stats rx */
4680         ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4681                            I40E_GLPRT_PRC64L(hw->port),
4682                            pf->stat_offsets_loaded,
4683                            &osd->rx_size_64, &nsd->rx_size_64);
4684         ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4685                            I40E_GLPRT_PRC127L(hw->port),
4686                            pf->stat_offsets_loaded,
4687                            &osd->rx_size_127, &nsd->rx_size_127);
4688         ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4689                            I40E_GLPRT_PRC255L(hw->port),
4690                            pf->stat_offsets_loaded,
4691                            &osd->rx_size_255, &nsd->rx_size_255);
4692         ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4693                            I40E_GLPRT_PRC511L(hw->port),
4694                            pf->stat_offsets_loaded,
4695                            &osd->rx_size_511, &nsd->rx_size_511);
4696         ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4697                            I40E_GLPRT_PRC1023L(hw->port),
4698                            pf->stat_offsets_loaded,
4699                            &osd->rx_size_1023, &nsd->rx_size_1023);
4700         ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4701                            I40E_GLPRT_PRC1522L(hw->port),
4702                            pf->stat_offsets_loaded,
4703                            &osd->rx_size_1522, &nsd->rx_size_1522);
4704         ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4705                            I40E_GLPRT_PRC9522L(hw->port),
4706                            pf->stat_offsets_loaded,
4707                            &osd->rx_size_big, &nsd->rx_size_big);
4708
4709         /* Packet size stats tx */
4710         ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4711                            I40E_GLPRT_PTC64L(hw->port),
4712                            pf->stat_offsets_loaded,
4713                            &osd->tx_size_64, &nsd->tx_size_64);
4714         ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4715                            I40E_GLPRT_PTC127L(hw->port),
4716                            pf->stat_offsets_loaded,
4717                            &osd->tx_size_127, &nsd->tx_size_127);
4718         ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4719                            I40E_GLPRT_PTC255L(hw->port),
4720                            pf->stat_offsets_loaded,
4721                            &osd->tx_size_255, &nsd->tx_size_255);
4722         ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4723                            I40E_GLPRT_PTC511L(hw->port),
4724                            pf->stat_offsets_loaded,
4725                            &osd->tx_size_511, &nsd->tx_size_511);
4726         ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4727                            I40E_GLPRT_PTC1023L(hw->port),
4728                            pf->stat_offsets_loaded,
4729                            &osd->tx_size_1023, &nsd->tx_size_1023);
4730         ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4731                            I40E_GLPRT_PTC1522L(hw->port),
4732                            pf->stat_offsets_loaded,
4733                            &osd->tx_size_1522, &nsd->tx_size_1522);
4734         ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4735                            I40E_GLPRT_PTC9522L(hw->port),
4736                            pf->stat_offsets_loaded,
4737                            &osd->tx_size_big, &nsd->tx_size_big);
4738
4739         ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4740                            pf->stat_offsets_loaded,
4741                            &osd->rx_undersize, &nsd->rx_undersize);
4742         ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4743                            pf->stat_offsets_loaded,
4744                            &osd->rx_fragments, &nsd->rx_fragments);
4745         ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4746                            pf->stat_offsets_loaded,
4747                            &osd->rx_oversize, &nsd->rx_oversize);
4748         ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4749                            pf->stat_offsets_loaded,
4750                            &osd->rx_jabber, &nsd->rx_jabber);
4751         pf->stat_offsets_loaded = true;
4752         /* End hw stats */
4753
4754         /* Update vsi stats */
4755         ixl_update_vsi_stats(vsi);
4756
4757         for (int i = 0; i < pf->num_vfs; i++) {
4758                 vf = &pf->vfs[i];
4759                 if (vf->vf_flags & VF_FLAG_ENABLED)
4760                         ixl_update_eth_stats(&pf->vfs[i].vsi);
4761         }
4762 }
4763
4764 static int
4765 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf)
4766 {
4767         struct i40e_hw *hw = &pf->hw;
4768         struct ixl_vsi *vsi = &pf->vsi;
4769         device_t dev = pf->dev;
4770         bool is_up = false;
4771         int error = 0;
4772
4773         is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING);
4774
4775         /* Teardown */
4776         if (is_up)
4777                 ixl_stop(pf);
4778         error = i40e_shutdown_lan_hmc(hw);
4779         if (error)
4780                 device_printf(dev,
4781                     "Shutdown LAN HMC failed with code %d\n", error);
4782         ixl_disable_adminq(hw);
4783         ixl_teardown_adminq_msix(pf);
4784         error = i40e_shutdown_adminq(hw);
4785         if (error)
4786                 device_printf(dev,
4787                     "Shutdown Admin queue failed with code %d\n", error);
4788
4789         /* Setup */
4790         error = i40e_init_adminq(hw);
4791         if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
4792                 device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
4793                     error);
4794         }
4795         error = ixl_setup_adminq_msix(pf);
4796         if (error) {
4797                 device_printf(dev, "ixl_setup_adminq_msix error: %d\n",
4798                     error);
4799         }
4800         ixl_configure_intr0_msix(pf);
4801         ixl_enable_adminq(hw);
4802         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4803             hw->func_caps.num_rx_qp, 0, 0);
4804         if (error) {
4805                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
4806         }
4807         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4808         if (error) {
4809                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
4810         }
4811         if (is_up)
4812                 ixl_init(pf);
4813
4814         return (0);
4815 }
4816
4817 static void
4818 ixl_handle_empr_reset(struct ixl_pf *pf)
4819 {
4820         struct i40e_hw *hw = &pf->hw;
4821         device_t dev = pf->dev;
4822         int count = 0;
4823         u32 reg;
4824
4825         /* Typically finishes within 3-4 seconds */
4826         while (count++ < 100) {
4827                 reg = rd32(hw, I40E_GLGEN_RSTAT)
4828                     & I40E_GLGEN_RSTAT_DEVSTATE_MASK;
4829                 if (reg)
4830                         i40e_msec_delay(100);
4831                 else
4832                         break;
4833         }
4834 #ifdef IXL_DEBUG
4835         // Reset-related
4836         device_printf(dev, "EMPR reset wait count: %d\n", count);
4837 #endif
4838
4839         device_printf(dev, "Rebuilding driver state...\n");
4840         ixl_rebuild_hw_structs_after_reset(pf);
4841         device_printf(dev, "Rebuilding driver state done.\n");
4842
4843         atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING);
4844 }
4845
4846 /*
4847 ** Tasklet handler for MSIX Adminq interrupts
4848 **  - do outside interrupt since it might sleep
4849 */
4850 static void
4851 ixl_do_adminq(void *context, int pending)
4852 {
4853         struct ixl_pf                   *pf = context;
4854         struct i40e_hw                  *hw = &pf->hw;
4855         struct i40e_arq_event_info      event;
4856         i40e_status                     ret;
4857         device_t                        dev = pf->dev;
4858         u32                             loop = 0;
4859         u16                             opcode, result;
4860
4861         if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
4862                 /* Flag cleared at end of this function */
4863                 ixl_handle_empr_reset(pf);
4864                 return;
4865         }
4866
4867         /* Admin Queue handling */
4868         event.buf_len = IXL_AQ_BUF_SZ;
4869         event.msg_buf = malloc(event.buf_len,
4870             M_DEVBUF, M_NOWAIT | M_ZERO);
4871         if (!event.msg_buf) {
4872                 device_printf(dev, "%s: Unable to allocate memory for Admin"
4873                     " Queue event!\n", __func__);
4874                 return;
4875         }
4876
4877         IXL_PF_LOCK(pf);
4878         /* clean and process any events */
4879         do {
4880                 ret = i40e_clean_arq_element(hw, &event, &result);
4881                 if (ret)
4882                         break;
4883                 opcode = LE16_TO_CPU(event.desc.opcode);
4884 #ifdef IXL_DEBUG
4885                 device_printf(dev, "%s: Admin Queue event: %#06x\n", __func__,
4886                     opcode);
4887 #endif
4888                 switch (opcode) {
4889                 case i40e_aqc_opc_get_link_status:
4890                         ixl_link_event(pf, &event);
4891                         break;
4892                 case i40e_aqc_opc_send_msg_to_pf:
4893 #ifdef PCI_IOV
4894                         ixl_handle_vf_msg(pf, &event);
4895 #endif
4896                         break;
4897                 case i40e_aqc_opc_event_lan_overflow:
4898                 default:
4899                         break;
4900                 }
4901
4902         } while (result && (loop++ < IXL_ADM_LIMIT));
4903
4904         free(event.msg_buf, M_DEVBUF);
4905
4906         /*
4907          * If there are still messages to process, reschedule ourselves.
4908          * Otherwise, re-enable our interrupt and go to sleep.
4909          */
4910         if (result > 0)
4911                 taskqueue_enqueue(pf->tq, &pf->adminq);
4912         else
4913                 ixl_enable_adminq(hw);
4914
4915         IXL_PF_UNLOCK(pf);
4916 }
4917
4918 /**
4919  * Update VSI-specific ethernet statistics counters.
4920  **/
4921 void
4922 ixl_update_eth_stats(struct ixl_vsi *vsi)
4923 {
4924         struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4925         struct i40e_hw *hw = &pf->hw;
4926         struct i40e_eth_stats *es;
4927         struct i40e_eth_stats *oes;
4928         struct i40e_hw_port_stats *nsd;
4929         u16 stat_idx = vsi->info.stat_counter_idx;
4930
4931         es = &vsi->eth_stats;
4932         oes = &vsi->eth_stats_offsets;
4933         nsd = &pf->stats;
4934
4935         /* Gather up the stats that the hw collects */
4936         ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4937                            vsi->stat_offsets_loaded,
4938                            &oes->tx_errors, &es->tx_errors);
4939         ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4940                            vsi->stat_offsets_loaded,
4941                            &oes->rx_discards, &es->rx_discards);
4942
4943         ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4944                            I40E_GLV_GORCL(stat_idx),
4945                            vsi->stat_offsets_loaded,
4946                            &oes->rx_bytes, &es->rx_bytes);
4947         ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4948                            I40E_GLV_UPRCL(stat_idx),
4949                            vsi->stat_offsets_loaded,
4950                            &oes->rx_unicast, &es->rx_unicast);
4951         ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4952                            I40E_GLV_MPRCL(stat_idx),
4953                            vsi->stat_offsets_loaded,
4954                            &oes->rx_multicast, &es->rx_multicast);
4955         ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4956                            I40E_GLV_BPRCL(stat_idx),
4957                            vsi->stat_offsets_loaded,
4958                            &oes->rx_broadcast, &es->rx_broadcast);
4959
4960         ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4961                            I40E_GLV_GOTCL(stat_idx),
4962                            vsi->stat_offsets_loaded,
4963                            &oes->tx_bytes, &es->tx_bytes);
4964         ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4965                            I40E_GLV_UPTCL(stat_idx),
4966                            vsi->stat_offsets_loaded,
4967                            &oes->tx_unicast, &es->tx_unicast);
4968         ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4969                            I40E_GLV_MPTCL(stat_idx),
4970                            vsi->stat_offsets_loaded,
4971                            &oes->tx_multicast, &es->tx_multicast);
4972         ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4973                            I40E_GLV_BPTCL(stat_idx),
4974                            vsi->stat_offsets_loaded,
4975                            &oes->tx_broadcast, &es->tx_broadcast);
4976         vsi->stat_offsets_loaded = true;
4977 }
4978
4979 static void
4980 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4981 {
4982         struct ixl_pf           *pf;
4983         struct ifnet            *ifp;
4984         struct i40e_eth_stats   *es;
4985         u64                     tx_discards;
4986
4987         struct i40e_hw_port_stats *nsd;
4988
4989         pf = vsi->back;
4990         ifp = vsi->ifp;
4991         es = &vsi->eth_stats;
4992         nsd = &pf->stats;
4993
4994         ixl_update_eth_stats(vsi);
4995
4996         tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4997         for (int i = 0; i < vsi->num_queues; i++)
4998                 tx_discards += vsi->queues[i].txr.br->br_drops;
4999
5000         /* Update ifnet stats */
5001         IXL_SET_IPACKETS(vsi, es->rx_unicast +
5002                            es->rx_multicast +
5003                            es->rx_broadcast);
5004         IXL_SET_OPACKETS(vsi, es->tx_unicast +
5005                            es->tx_multicast +
5006                            es->tx_broadcast);
5007         IXL_SET_IBYTES(vsi, es->rx_bytes);
5008         IXL_SET_OBYTES(vsi, es->tx_bytes);
5009         IXL_SET_IMCASTS(vsi, es->rx_multicast);
5010         IXL_SET_OMCASTS(vsi, es->tx_multicast);
5011
5012         IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
5013             nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
5014             nsd->rx_jabber);
5015         IXL_SET_OERRORS(vsi, es->tx_errors);
5016         IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
5017         IXL_SET_OQDROPS(vsi, tx_discards);
5018         IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
5019         IXL_SET_COLLISIONS(vsi, 0);
5020 }
5021
5022 /**
5023  * Reset all of the stats for the given pf
5024  **/
5025 void ixl_pf_reset_stats(struct ixl_pf *pf)
5026 {
5027         bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
5028         bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
5029         pf->stat_offsets_loaded = false;
5030 }
5031
5032 /**
5033  * Resets all stats of the given vsi
5034  **/
5035 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
5036 {
5037         bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
5038         bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
5039         vsi->stat_offsets_loaded = false;
5040 }
5041
5042 /**
5043  * Read and update a 48 bit stat from the hw
5044  *
5045  * Since the device stats are not reset at PFReset, they likely will not
5046  * be zeroed when the driver starts.  We'll save the first values read
5047  * and use them as offsets to be subtracted from the raw values in order
5048  * to report stats that count from zero.
5049  **/
5050 static void
5051 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
5052         bool offset_loaded, u64 *offset, u64 *stat)
5053 {
5054         u64 new_data;
5055
5056 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
5057         new_data = rd64(hw, loreg);
5058 #else
5059         /*
5060          * Use two rd32's instead of one rd64; FreeBSD versions before
5061          * 10 don't support 8 byte bus reads/writes.
5062          */
5063         new_data = rd32(hw, loreg);
5064         new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
5065 #endif
5066
5067         if (!offset_loaded)
5068                 *offset = new_data;
5069         if (new_data >= *offset)
5070                 *stat = new_data - *offset;
5071         else
5072                 *stat = (new_data + ((u64)1 << 48)) - *offset;
5073         *stat &= 0xFFFFFFFFFFFFULL;
5074 }
5075
5076 /**
5077  * Read and update a 32 bit stat from the hw
5078  **/
5079 static void
5080 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
5081         bool offset_loaded, u64 *offset, u64 *stat)
5082 {
5083         u32 new_data;
5084
5085         new_data = rd32(hw, reg);
5086         if (!offset_loaded)
5087                 *offset = new_data;
5088         if (new_data >= *offset)
5089                 *stat = (u32)(new_data - *offset);
5090         else
5091                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
5092 }
5093
5094 static void
5095 ixl_add_device_sysctls(struct ixl_pf *pf)
5096 {
5097         device_t dev = pf->dev;
5098
5099         /* Set up sysctls */
5100         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5101             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5102             OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
5103             pf, 0, ixl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC);
5104
5105         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5106             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5107             OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
5108             pf, 0, ixl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE);
5109
5110         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5111             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5112             OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
5113             pf, 0, ixl_current_speed, "A", "Current Port Speed");
5114
5115         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5116             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5117             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
5118             pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
5119
5120 #if 0
5121         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5122             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5123             OID_AUTO, "rx_itr", CTLFLAG_RW,
5124             &ixl_rx_itr, 0, "RX ITR");
5125
5126         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5127             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5128             OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
5129             &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
5130
5131         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5132             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5133             OID_AUTO, "tx_itr", CTLFLAG_RW,
5134             &ixl_tx_itr, 0, "TX ITR");
5135
5136         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5137             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5138             OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
5139             &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
5140 #endif
5141
5142 #ifdef IXL_DEBUG_SYSCTL
5143         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5144             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5145             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
5146             ixl_debug_info, "I", "Debug Information");
5147
5148         /* Shared-code debug message level */
5149         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5150             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5151             OID_AUTO, "debug_mask", CTLFLAG_RW,
5152             &pf->hw.debug_mask, 0, "Debug Message Level");
5153
5154         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5155             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5156             OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
5157             pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS);
5158
5159         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5160             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5161             OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
5162             pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
5163
5164         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5165             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5166             OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
5167             pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
5168
5169         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5170             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5171             OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
5172             pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
5173
5174         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
5175             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5176             OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
5177             pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
5178
5179 #ifdef PCI_IOV
5180         SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
5181             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
5182             OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
5183             0, "PF/VF Virtual Channel debug level");
5184 #endif
5185 #endif
5186 }
5187
5188 /*
5189 ** Set flow control using sysctl:
5190 **      0 - off
5191 **      1 - rx pause
5192 **      2 - tx pause
5193 **      3 - full
5194 */
5195 static int
5196 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
5197 {
5198         /*
5199          * TODO: ensure tx CRC by hardware should be enabled
5200          * if tx flow control is enabled.
5201          * ^ N/A for 40G ports
5202          */
5203         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5204         struct i40e_hw *hw = &pf->hw;
5205         device_t dev = pf->dev;
5206         int requested_fc, error = 0;
5207         enum i40e_status_code aq_error = 0;
5208         u8 fc_aq_err = 0;
5209
5210         /* Get request */
5211         requested_fc = pf->fc;
5212         error = sysctl_handle_int(oidp, &requested_fc, 0, req);
5213         if ((error) || (req->newptr == NULL))
5214                 return (error);
5215         if (requested_fc < 0 || requested_fc > 3) {
5216                 device_printf(dev,
5217                     "Invalid fc mode; valid modes are 0 through 3\n");
5218                 return (EINVAL);
5219         }
5220
5221         /* Set fc ability for port */
5222         hw->fc.requested_mode = requested_fc;
5223         aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
5224         if (aq_error) {
5225                 device_printf(dev,
5226                     "%s: Error setting new fc mode %d; fc_err %#x\n",
5227                     __func__, aq_error, fc_aq_err);
5228                 return (EIO);
5229         }
5230         pf->fc = requested_fc;
5231
5232         /* Get new link state */
5233         i40e_msec_delay(250);
5234         hw->phy.get_link_info = TRUE;
5235         i40e_get_link_status(hw, &pf->link_up);
5236
5237         return (0);
5238 }
5239
5240 static int
5241 ixl_current_speed(SYSCTL_HANDLER_ARGS)
5242 {
5243         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5244         struct i40e_hw *hw = &pf->hw;
5245         int error = 0, index = 0;
5246
5247         char *speeds[] = {
5248                 "Unknown",
5249                 "100M",
5250                 "1G",
5251                 "10G",
5252                 "40G",
5253                 "20G"
5254         };
5255
5256         ixl_update_link_status(pf);
5257
5258         switch (hw->phy.link_info.link_speed) {
5259         case I40E_LINK_SPEED_100MB:
5260                 index = 1;
5261                 break;
5262         case I40E_LINK_SPEED_1GB:
5263                 index = 2;
5264                 break;
5265         case I40E_LINK_SPEED_10GB:
5266                 index = 3;
5267                 break;
5268         case I40E_LINK_SPEED_40GB:
5269                 index = 4;
5270                 break;
5271         case I40E_LINK_SPEED_20GB:
5272                 index = 5;
5273                 break;
5274         case I40E_LINK_SPEED_UNKNOWN:
5275         default:
5276                 index = 0;
5277                 break;
5278         }
5279
5280         error = sysctl_handle_string(oidp, speeds[index],
5281             strlen(speeds[index]), req);
5282         return (error);
5283 }
5284
5285 static int
5286 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
5287 {
5288         struct i40e_hw *hw = &pf->hw;
5289         device_t dev = pf->dev;
5290         struct i40e_aq_get_phy_abilities_resp abilities;
5291         struct i40e_aq_set_phy_config config;
5292         enum i40e_status_code aq_error = 0;
5293
5294         /* Get current capability information */
5295         aq_error = i40e_aq_get_phy_capabilities(hw,
5296             FALSE, FALSE, &abilities, NULL);
5297         if (aq_error) {
5298                 device_printf(dev,
5299                     "%s: Error getting phy capabilities %d,"
5300                     " aq error: %d\n", __func__, aq_error,
5301                     hw->aq.asq_last_status);
5302                 return (EAGAIN);
5303         }
5304
5305         /* Prepare new config */
5306         bzero(&config, sizeof(config));
5307         config.phy_type = abilities.phy_type;
5308         config.abilities = abilities.abilities
5309             | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
5310         config.eee_capability = abilities.eee_capability;
5311         config.eeer = abilities.eeer_val;
5312         config.low_power_ctrl = abilities.d3_lpan;
5313         /* Translate into aq cmd link_speed */
5314         if (speeds & 0x10)
5315                 config.link_speed |= I40E_LINK_SPEED_40GB;
5316         if (speeds & 0x8)
5317                 config.link_speed |= I40E_LINK_SPEED_20GB;
5318         if (speeds & 0x4)
5319                 config.link_speed |= I40E_LINK_SPEED_10GB;
5320         if (speeds & 0x2)
5321                 config.link_speed |= I40E_LINK_SPEED_1GB;
5322         if (speeds & 0x1)
5323                 config.link_speed |= I40E_LINK_SPEED_100MB;
5324
5325         /* Do aq command & restart link */
5326         aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
5327         if (aq_error) {
5328                 device_printf(dev,
5329                     "%s: Error setting new phy config %d,"
5330                     " aq error: %d\n", __func__, aq_error,
5331                     hw->aq.asq_last_status);
5332                 return (EAGAIN);
5333         }
5334
5335         /*
5336         ** This seems a bit heavy handed, but we
5337         ** need to get a reinit on some devices
5338         */
5339         IXL_PF_LOCK(pf);
5340         ixl_stop_locked(pf);
5341         ixl_init_locked(pf);
5342         IXL_PF_UNLOCK(pf);
5343
5344         return (0);
5345 }
5346
5347 /*
5348 ** Control link advertise speed:
5349 **      Flags:
5350 **       0x1 - advertise 100 Mb
5351 **       0x2 - advertise 1G
5352 **       0x4 - advertise 10G
5353 **       0x8 - advertise 20G
5354 **      0x10 - advertise 40G
5355 **
5356 **      Set to 0 to disable link
5357 */
5358 static int
5359 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
5360 {
5361         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5362         struct i40e_hw *hw = &pf->hw;
5363         device_t dev = pf->dev;
5364         int requested_ls = 0;
5365         int error = 0;
5366
5367         /* Read in new mode */
5368         requested_ls = pf->advertised_speed;
5369         error = sysctl_handle_int(oidp, &requested_ls, 0, req);
5370         if ((error) || (req->newptr == NULL))
5371                 return (error);
5372         /* Check for sane value */
5373         if (requested_ls > 0x10) {
5374                 device_printf(dev, "Invalid advertised speed; "
5375                     "valid modes are 0x1 through 0x10\n");
5376                 return (EINVAL);
5377         }
5378         /* Then check for validity based on adapter type */
5379         switch (hw->device_id) {
5380         case I40E_DEV_ID_10G_BASE_T:
5381         case I40E_DEV_ID_10G_BASE_T4:
5382                 /* BaseT */
5383                 if (requested_ls & ~(0x7)) {
5384                         device_printf(dev,
5385                             "Only 100M/1G/10G speeds supported on this device.\n");
5386                         return (EINVAL);
5387                 }
5388                 break;
5389         case I40E_DEV_ID_20G_KR2:
5390         case I40E_DEV_ID_20G_KR2_A:
5391                 /* 20G */
5392                 if (requested_ls & ~(0xE)) {
5393                         device_printf(dev,
5394                             "Only 1G/10G/20G speeds supported on this device.\n");
5395                         return (EINVAL);
5396                 }
5397                 break;
5398         case I40E_DEV_ID_KX_B:
5399         case I40E_DEV_ID_QSFP_A:
5400         case I40E_DEV_ID_QSFP_B:
5401                 /* 40G */
5402                 if (requested_ls & ~(0x10)) {
5403                         device_printf(dev,
5404                             "Only 40G speeds supported on this device.\n");
5405                         return (EINVAL);
5406                 }
5407                 break;
5408         default:
5409                 /* 10G (1G) */
5410                 if (requested_ls & ~(0x6)) {
5411                         device_printf(dev,
5412                             "Only 1/10Gbs speeds are supported on this device.\n");
5413                         return (EINVAL);
5414                 }
5415                 break;
5416         }
5417
5418         /* Exit if no change */
5419         if (pf->advertised_speed == requested_ls)
5420                 return (0);
5421
5422         error = ixl_set_advertised_speeds(pf, requested_ls);
5423         if (error)
5424                 return (error);
5425
5426         pf->advertised_speed = requested_ls;
5427         ixl_update_link_status(pf);
5428         return (0);
5429 }
5430
5431 /*
5432 ** Get the width and transaction speed of
5433 ** the bus this adapter is plugged into.
5434 */
5435 static u16
5436 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
5437 {
5438         u16                     link;
5439         u32                     offset;
5440
5441         /* Get the PCI Express Capabilities offset */
5442         pci_find_cap(dev, PCIY_EXPRESS, &offset);
5443
5444         /* ...and read the Link Status Register */
5445         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
5446
5447         switch (link & I40E_PCI_LINK_WIDTH) {
5448         case I40E_PCI_LINK_WIDTH_1:
5449                 hw->bus.width = i40e_bus_width_pcie_x1;
5450                 break;
5451         case I40E_PCI_LINK_WIDTH_2:
5452                 hw->bus.width = i40e_bus_width_pcie_x2;
5453                 break;
5454         case I40E_PCI_LINK_WIDTH_4:
5455                 hw->bus.width = i40e_bus_width_pcie_x4;
5456                 break;
5457         case I40E_PCI_LINK_WIDTH_8:
5458                 hw->bus.width = i40e_bus_width_pcie_x8;
5459                 break;
5460         default:
5461                 hw->bus.width = i40e_bus_width_unknown;
5462                 break;
5463         }
5464
5465         switch (link & I40E_PCI_LINK_SPEED) {
5466         case I40E_PCI_LINK_SPEED_2500:
5467                 hw->bus.speed = i40e_bus_speed_2500;
5468                 break;
5469         case I40E_PCI_LINK_SPEED_5000:
5470                 hw->bus.speed = i40e_bus_speed_5000;
5471                 break;
5472         case I40E_PCI_LINK_SPEED_8000:
5473                 hw->bus.speed = i40e_bus_speed_8000;
5474                 break;
5475         default:
5476                 hw->bus.speed = i40e_bus_speed_unknown;
5477                 break;
5478         }
5479
5480         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
5481             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
5482             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
5483             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
5484             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
5485             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
5486             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
5487             ("Unknown"));
5488
5489         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
5490             (hw->bus.speed < i40e_bus_speed_8000)) {
5491                 device_printf(dev, "PCI-Express bandwidth available"
5492                     " for this device\n     may be insufficient for"
5493                     " optimal performance.\n");
5494                 device_printf(dev, "For expected performance a x8 "
5495                     "PCIE Gen3 slot is required.\n");
5496         }
5497
5498         return (link);
5499 }
5500
5501 static int
5502 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
5503 {
5504         struct ixl_pf   *pf = (struct ixl_pf *)arg1;
5505         struct i40e_hw  *hw = &pf->hw;
5506         struct sbuf     *sbuf;
5507
5508         sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5509         ixl_nvm_version_str(hw, sbuf);
5510         sbuf_finish(sbuf);
5511         sbuf_delete(sbuf);
5512
5513         return 0;
5514 }
5515
5516 #ifdef IXL_DEBUG
5517 static void
5518 ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma)
5519 {
5520         if ((nvma->command == I40E_NVM_READ) &&
5521             ((nvma->config & 0xFF) == 0xF) &&
5522             (((nvma->config & 0xF00) >> 8) == 0xF) &&
5523             (nvma->offset == 0) &&
5524             (nvma->data_size == 1)) {
5525                 // device_printf(dev, "- Get Driver Status Command\n");
5526         }
5527         else if (nvma->command == I40E_NVM_READ) {
5528
5529         }
5530         else {
5531                 switch (nvma->command) {
5532                 case 0xB:
5533                         device_printf(dev, "- command: I40E_NVM_READ\n");
5534                         break;
5535                 case 0xC:
5536                         device_printf(dev, "- command: I40E_NVM_WRITE\n");
5537                         break;
5538                 default:
5539                         device_printf(dev, "- command: unknown 0x%08x\n", nvma->command);
5540                         break;
5541                 }
5542
5543                 device_printf(dev, "- config (ptr)  : 0x%02x\n", nvma->config & 0xFF);
5544                 device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8);
5545                 device_printf(dev, "- offset : 0x%08x\n", nvma->offset);
5546                 device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size);
5547         }
5548 }
5549 #endif
5550
5551 static int
5552 ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd)
5553 {
5554         struct i40e_hw *hw = &pf->hw;
5555         struct i40e_nvm_access *nvma;
5556         device_t dev = pf->dev;
5557         enum i40e_status_code status = 0;
5558         int perrno;
5559
5560         DEBUGFUNC("ixl_handle_nvmupd_cmd");
5561
5562         /* Sanity checks */
5563         if (ifd->ifd_len < sizeof(struct i40e_nvm_access) ||
5564             ifd->ifd_data == NULL) {
5565                 device_printf(dev, "%s: incorrect ifdrv length or data pointer\n",
5566                     __func__);
5567                 device_printf(dev, "%s: ifdrv length: %lu, sizeof(struct i40e_nvm_access): %lu\n",
5568                     __func__, ifd->ifd_len, sizeof(struct i40e_nvm_access));
5569                 device_printf(dev, "%s: data pointer: %p\n", __func__,
5570                     ifd->ifd_data);
5571                 return (EINVAL);
5572         }
5573
5574         nvma = (struct i40e_nvm_access *)ifd->ifd_data;
5575
5576 #ifdef IXL_DEBUG
5577         ixl_print_nvm_cmd(dev, nvma);
5578 #endif
5579
5580         if (pf->state & IXL_PF_STATE_EMPR_RESETTING) {
5581                 int count = 0;
5582                 while (count++ < 100) {
5583                         i40e_msec_delay(100);
5584                         if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING))
5585                                 break;
5586                 }
5587         }
5588
5589         if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) {
5590                 IXL_PF_LOCK(pf);
5591                 status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno);
5592                 IXL_PF_UNLOCK(pf);
5593         } else {
5594                 perrno = -EBUSY;
5595         }
5596
5597         if (status)
5598                 device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n",
5599                     status, perrno);
5600
5601         /*
5602          * -EPERM is actually ERESTART, which the kernel interprets as it needing
5603          * to run this ioctl again. So use -EACCES for -EPERM instead.
5604          */
5605         if (perrno == -EPERM)
5606                 return (-EACCES);
5607         else
5608                 return (perrno);
5609 }
5610
5611 #ifdef IXL_DEBUG_SYSCTL
5612 static int
5613 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
5614 {
5615         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5616         struct i40e_hw *hw = &pf->hw;
5617         struct i40e_link_status link_status;
5618         char buf[512];
5619
5620         enum i40e_status_code aq_error = 0;
5621
5622         aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
5623         if (aq_error) {
5624                 printf("i40e_aq_get_link_info() error %d\n", aq_error);
5625                 return (EPERM);
5626         }
5627
5628         sprintf(buf, "\n"
5629             "PHY Type : %#04x\n"
5630             "Speed    : %#04x\n"
5631             "Link info: %#04x\n"
5632             "AN info  : %#04x\n"
5633             "Ext info : %#04x\n"
5634             "Max Frame: %d\n"
5635             "Pacing   : %#04x\n"
5636             "CRC En?  : %d",
5637             link_status.phy_type, link_status.link_speed,
5638             link_status.link_info, link_status.an_info,
5639             link_status.ext_info, link_status.max_frame_size,
5640             link_status.pacing, link_status.crc_enable);
5641
5642         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5643 }
5644
5645 static int
5646 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
5647 {
5648         struct ixl_pf           *pf = (struct ixl_pf *)arg1;
5649         struct i40e_hw          *hw = &pf->hw;
5650         char                    buf[512];
5651         enum i40e_status_code   aq_error = 0;
5652
5653         struct i40e_aq_get_phy_abilities_resp abilities;
5654
5655         aq_error = i40e_aq_get_phy_capabilities(hw,
5656             TRUE, FALSE, &abilities, NULL);
5657         if (aq_error) {
5658                 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
5659                 return (EPERM);
5660         }
5661
5662         sprintf(buf, "\n"
5663             "PHY Type : %#010x\n"
5664             "Speed    : %#04x\n"
5665             "Abilities: %#04x\n"
5666             "EEE cap  : %#06x\n"
5667             "EEER reg : %#010x\n"
5668             "D3 Lpan  : %#04x",
5669             abilities.phy_type, abilities.link_speed,
5670             abilities.abilities, abilities.eee_capability,
5671             abilities.eeer_val, abilities.d3_lpan);
5672
5673         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5674 }
5675
5676 static int
5677 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5678 {
5679         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5680         struct ixl_vsi *vsi = &pf->vsi;
5681         struct ixl_mac_filter *f;
5682         char *buf, *buf_i;
5683
5684         int error = 0;
5685         int ftl_len = 0;
5686         int ftl_counter = 0;
5687         int buf_len = 0;
5688         int entry_len = 42;
5689
5690         SLIST_FOREACH(f, &vsi->ftl, next) {
5691                 ftl_len++;
5692         }
5693
5694         if (ftl_len < 1) {
5695                 sysctl_handle_string(oidp, "(none)", 6, req);
5696                 return (0);
5697         }
5698
5699         buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5700         buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5701
5702         sprintf(buf_i++, "\n");
5703         SLIST_FOREACH(f, &vsi->ftl, next) {
5704                 sprintf(buf_i,
5705                     MAC_FORMAT ", vlan %4d, flags %#06x",
5706                     MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5707                 buf_i += entry_len;
5708                 /* don't print '\n' for last entry */
5709                 if (++ftl_counter != ftl_len) {
5710                         sprintf(buf_i, "\n");
5711                         buf_i++;
5712                 }
5713         }
5714
5715         error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5716         if (error)
5717                 printf("sysctl error: %d\n", error);
5718         free(buf, M_DEVBUF);
5719         return error;
5720 }
5721
5722 #define IXL_SW_RES_SIZE 0x14
5723 static int
5724 ixl_res_alloc_cmp(const void *a, const void *b)
5725 {
5726         const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5727         one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5728         two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5729
5730         return ((int)one->resource_type - (int)two->resource_type);
5731 }
5732
5733 /*
5734  * Longest string length: 25
5735  */
5736 static char *
5737 ixl_switch_res_type_string(u8 type)
5738 {
5739         static char * ixl_switch_res_type_strings[0x14] = {
5740                 "VEB",
5741                 "VSI",
5742                 "Perfect Match MAC address",
5743                 "S-tag",
5744                 "(Reserved)",
5745                 "Multicast hash entry",
5746                 "Unicast hash entry",
5747                 "VLAN",
5748                 "VSI List entry",
5749                 "(Reserved)",
5750                 "VLAN Statistic Pool",
5751                 "Mirror Rule",
5752                 "Queue Set",
5753                 "Inner VLAN Forward filter",
5754                 "(Reserved)",
5755                 "Inner MAC",
5756                 "IP",
5757                 "GRE/VN1 Key",
5758                 "VN2 Key",
5759                 "Tunneling Port"
5760         };
5761
5762         if (type < 0x14)
5763                 return ixl_switch_res_type_strings[type];
5764         else
5765                 return "(Reserved)";
5766 }
5767
5768 static int
5769 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5770 {
5771         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5772         struct i40e_hw *hw = &pf->hw;
5773         device_t dev = pf->dev;
5774         struct sbuf *buf;
5775         int error = 0;
5776
5777         u8 num_entries;
5778         struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5779
5780         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5781         if (!buf) {
5782                 device_printf(dev, "Could not allocate sbuf for output.\n");
5783                 return (ENOMEM);
5784         }
5785
5786         bzero(resp, sizeof(resp));
5787         error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5788                                 resp,
5789                                 IXL_SW_RES_SIZE,
5790                                 NULL);
5791         if (error) {
5792                 device_printf(dev,
5793                     "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5794                     __func__, error, hw->aq.asq_last_status);
5795                 sbuf_delete(buf);
5796                 return error;
5797         }
5798
5799         /* Sort entries by type for display */
5800         qsort(resp, num_entries,
5801             sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5802             &ixl_res_alloc_cmp);
5803
5804         sbuf_cat(buf, "\n");
5805         sbuf_printf(buf, "# of entries: %d\n", num_entries);
5806         sbuf_printf(buf,
5807             "                     Type | Guaranteed | Total | Used   | Un-allocated\n"
5808             "                          | (this)     | (all) | (this) | (all)       \n");
5809         for (int i = 0; i < num_entries; i++) {
5810                 sbuf_printf(buf,
5811                     "%25s | %10d   %5d   %6d   %12d",
5812                     ixl_switch_res_type_string(resp[i].resource_type),
5813                     resp[i].guaranteed,
5814                     resp[i].total,
5815                     resp[i].used,
5816                     resp[i].total_unalloced);
5817                 if (i < num_entries - 1)
5818                         sbuf_cat(buf, "\n");
5819         }
5820
5821         error = sbuf_finish(buf);
5822         if (error)
5823                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5824
5825         sbuf_delete(buf);
5826         return error;
5827 }
5828
5829 /*
5830 ** Caller must init and delete sbuf; this function will clear and
5831 ** finish it for caller.
5832 **
5833 ** XXX: Cannot use the SEID for this, since there is no longer a
5834 ** fixed mapping between SEID and element type.
5835 */
5836 static char *
5837 ixl_switch_element_string(struct sbuf *s,
5838     struct i40e_aqc_switch_config_element_resp *element)
5839 {
5840         sbuf_clear(s);
5841
5842         switch (element->element_type) {
5843         case I40E_AQ_SW_ELEM_TYPE_MAC:
5844                 sbuf_printf(s, "MAC %3d", element->element_info);
5845                 break;
5846         case I40E_AQ_SW_ELEM_TYPE_PF:
5847                 sbuf_printf(s, "PF  %3d", element->element_info);
5848                 break;
5849         case I40E_AQ_SW_ELEM_TYPE_VF:
5850                 sbuf_printf(s, "VF  %3d", element->element_info);
5851                 break;
5852         case I40E_AQ_SW_ELEM_TYPE_EMP:
5853                 sbuf_cat(s, "EMP");
5854                 break;
5855         case I40E_AQ_SW_ELEM_TYPE_BMC:
5856                 sbuf_cat(s, "BMC");
5857                 break;
5858         case I40E_AQ_SW_ELEM_TYPE_PV:
5859                 sbuf_cat(s, "PV");
5860                 break;
5861         case I40E_AQ_SW_ELEM_TYPE_VEB:
5862                 sbuf_cat(s, "VEB");
5863                 break;
5864         case I40E_AQ_SW_ELEM_TYPE_PA:
5865                 sbuf_cat(s, "PA");
5866                 break;
5867         case I40E_AQ_SW_ELEM_TYPE_VSI:
5868                 sbuf_printf(s, "VSI %3d", element->element_info);
5869                 break;
5870         default:
5871                 sbuf_cat(s, "?");
5872                 break;
5873         }
5874
5875         sbuf_finish(s);
5876         return sbuf_data(s);
5877 }
5878
5879 static int
5880 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5881 {
5882         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5883         struct i40e_hw *hw = &pf->hw;
5884         device_t dev = pf->dev;
5885         struct sbuf *buf;
5886         struct sbuf *nmbuf;
5887         int error = 0;
5888         u16 next = 0;
5889         u8 aq_buf[I40E_AQ_LARGE_BUF];
5890
5891         struct i40e_aqc_get_switch_config_resp *sw_config;
5892         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5893
5894         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5895         if (!buf) {
5896                 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5897                 return (ENOMEM);
5898         }
5899
5900         error = i40e_aq_get_switch_config(hw, sw_config,
5901             sizeof(aq_buf), &next, NULL);
5902         if (error) {
5903                 device_printf(dev,
5904                     "%s: aq_get_switch_config() error %d, aq error %d\n",
5905                     __func__, error, hw->aq.asq_last_status);
5906                 sbuf_delete(buf);
5907                 return error;
5908         }
5909         if (next)
5910                 device_printf(dev, "%s: TODO: get more config with SEID %d\n",
5911                     __func__, next);
5912
5913         nmbuf = sbuf_new_auto();
5914         if (!nmbuf) {
5915                 device_printf(dev, "Could not allocate sbuf for name output.\n");
5916                 sbuf_delete(buf);
5917                 return (ENOMEM);
5918         }
5919
5920         sbuf_cat(buf, "\n");
5921         // Assuming <= 255 elements in switch
5922         sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported);
5923         sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total);
5924         /* Exclude:
5925         ** Revision -- all elements are revision 1 for now
5926         */
5927         sbuf_printf(buf,
5928             "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5929             "                |          |          | (uplink)\n");
5930         for (int i = 0; i < sw_config->header.num_reported; i++) {
5931                 // "%4d (%8s) | %8s   %8s   %#8x",
5932                 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5933                 sbuf_cat(buf, " ");
5934                 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5935                     &sw_config->element[i]));
5936                 sbuf_cat(buf, " | ");
5937                 sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid);
5938                 sbuf_cat(buf, "   ");
5939                 sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid);
5940                 sbuf_cat(buf, "   ");
5941                 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5942                 if (i < sw_config->header.num_reported - 1)
5943                         sbuf_cat(buf, "\n");
5944         }
5945         sbuf_delete(nmbuf);
5946
5947         error = sbuf_finish(buf);
5948         if (error)
5949                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5950
5951         sbuf_delete(buf);
5952
5953         return (error);
5954 }
5955
5956 static int
5957 ixl_debug_info(SYSCTL_HANDLER_ARGS)
5958 {
5959         struct ixl_pf   *pf;
5960         int             error, input = 0;
5961
5962         error = sysctl_handle_int(oidp, &input, 0, req);
5963
5964         if (error || !req->newptr)
5965                 return (error);
5966
5967         if (input == 1) {
5968                 pf = (struct ixl_pf *)arg1;
5969                 ixl_print_debug_info(pf);
5970         }
5971
5972         return (error);
5973 }
5974
5975 static void
5976 ixl_print_debug_info(struct ixl_pf *pf)
5977 {
5978         struct i40e_hw          *hw = &pf->hw;
5979         struct ixl_vsi          *vsi = &pf->vsi;
5980         struct ixl_queue        *que = vsi->queues;
5981         struct rx_ring          *rxr = &que->rxr;
5982         struct tx_ring          *txr = &que->txr;
5983         u32                     reg;
5984
5985
5986         printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
5987         printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
5988         printf("RX next check = %x\n", rxr->next_check);
5989         printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
5990         printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
5991         printf("TX desc avail = %x\n", txr->avail);
5992
5993         reg = rd32(hw, I40E_GLV_GORCL(0xc));
5994          printf("RX Bytes = %x\n", reg);
5995         reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
5996          printf("Port RX Bytes = %x\n", reg);
5997         reg = rd32(hw, I40E_GLV_RDPC(0xc));
5998          printf("RX discard = %x\n", reg);
5999         reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
6000          printf("Port RX discard = %x\n", reg);
6001
6002         reg = rd32(hw, I40E_GLV_TEPC(0xc));
6003          printf("TX errors = %x\n", reg);
6004         reg = rd32(hw, I40E_GLV_GOTCL(0xc));
6005          printf("TX Bytes = %x\n", reg);
6006
6007         reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
6008          printf("RX undersize = %x\n", reg);
6009         reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
6010          printf("RX fragments = %x\n", reg);
6011         reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
6012          printf("RX oversize = %x\n", reg);
6013         reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
6014          printf("RX length error = %x\n", reg);
6015         reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
6016          printf("mac remote fault = %x\n", reg);
6017         reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
6018          printf("mac local fault = %x\n", reg);
6019 }
6020
6021 #endif /* IXL_DEBUG_SYSCTL */
6022
6023 #ifdef PCI_IOV
6024 static int
6025 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6026 {
6027         struct i40e_hw *hw;
6028         struct ixl_vsi *vsi;
6029         struct i40e_vsi_context vsi_ctx;
6030         int i;
6031         uint16_t first_queue;
6032         enum i40e_status_code code;
6033
6034         hw = &pf->hw;
6035         vsi = &pf->vsi;
6036
6037         vsi_ctx.pf_num = hw->pf_id;
6038         vsi_ctx.uplink_seid = pf->veb_seid;
6039         vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
6040         vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6041         vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
6042
6043         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6044
6045         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6046         vsi_ctx.info.switch_id = htole16(0);
6047
6048         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
6049         vsi_ctx.info.sec_flags = 0;
6050         if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
6051                 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
6052
6053         /* TODO: If a port VLAN is set, then this needs to be changed */
6054         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6055         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6056             I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
6057
6058         vsi_ctx.info.valid_sections |=
6059             htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
6060         vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
6061         first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
6062         for (i = 0; i < IXLV_MAX_QUEUES; i++)
6063                 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
6064         for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
6065                 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
6066
6067         vsi_ctx.info.tc_mapping[0] = htole16(
6068             (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6069             (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
6070
6071         code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
6072         if (code != I40E_SUCCESS)
6073                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6074         vf->vsi.seid = vsi_ctx.seid;
6075         vf->vsi.vsi_num = vsi_ctx.vsi_number;
6076         vf->vsi.first_queue = first_queue;
6077         vf->vsi.num_queues = IXLV_MAX_QUEUES;
6078
6079         code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
6080         if (code != I40E_SUCCESS)
6081                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6082
6083         code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
6084         if (code != I40E_SUCCESS) {
6085                 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
6086                     ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6087                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
6088         }
6089
6090         memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
6091         return (0);
6092 }
6093
6094 static int
6095 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
6096 {
6097         struct i40e_hw *hw;
6098         int error;
6099
6100         hw = &pf->hw;
6101
6102         error = ixl_vf_alloc_vsi(pf, vf);
6103         if (error != 0)
6104                 return (error);
6105
6106         vf->vsi.hw_filters_add = 0;
6107         vf->vsi.hw_filters_del = 0;
6108         ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
6109         ixl_reconfigure_filters(&vf->vsi);
6110
6111         return (0);
6112 }
6113
6114 static void
6115 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
6116     uint32_t val)
6117 {
6118         uint32_t qtable;
6119         int index, shift;
6120
6121         /*
6122          * Two queues are mapped in a single register, so we have to do some
6123          * gymnastics to convert the queue number into a register index and
6124          * shift.
6125          */
6126         index = qnum / 2;
6127         shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
6128
6129         qtable = i40e_read_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
6130         qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
6131         qtable |= val << shift;
6132         i40e_write_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
6133 }
6134
6135 static void
6136 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
6137 {
6138         struct i40e_hw *hw;
6139         uint32_t qtable;
6140         int i;
6141
6142         hw = &pf->hw;
6143
6144         /*
6145          * Contiguous mappings aren't actually supported by the hardware,
6146          * so we have to use non-contiguous mappings.
6147          */
6148         i40e_write_rx_ctl(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
6149              I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
6150
6151         wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
6152             I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
6153
6154         for (i = 0; i < vf->vsi.num_queues; i++) {
6155                 qtable = (vf->vsi.first_queue + i) <<
6156                     I40E_VPLAN_QTABLE_QINDEX_SHIFT;
6157
6158                 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
6159         }
6160
6161         /* Map queues allocated to VF to its VSI. */
6162         for (i = 0; i < vf->vsi.num_queues; i++)
6163                 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
6164
6165         /* Set rest of VSI queues as unused. */
6166         for (; i < IXL_MAX_VSI_QUEUES; i++)
6167                 ixl_vf_map_vsi_queue(hw, vf, i,
6168                     I40E_VSILAN_QTABLE_QINDEX_0_MASK);
6169
6170         ixl_flush(hw);
6171 }
6172
6173 static void
6174 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
6175 {
6176         struct i40e_hw *hw;
6177
6178         hw = &pf->hw;
6179
6180         if (vsi->seid == 0)
6181                 return;
6182
6183         i40e_aq_delete_element(hw, vsi->seid, NULL);
6184 }
6185
6186 static void
6187 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
6188 {
6189
6190         wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
6191         ixl_flush(hw);
6192 }
6193
6194 static void
6195 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
6196 {
6197
6198         wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
6199             I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
6200         ixl_flush(hw);
6201 }
6202
6203 static void
6204 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
6205 {
6206         struct i40e_hw *hw;
6207         uint32_t vfint_reg, vpint_reg;
6208         int i;
6209
6210         hw = &pf->hw;
6211
6212         ixl_vf_vsi_release(pf, &vf->vsi);
6213
6214         /* Index 0 has a special register. */
6215         ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
6216
6217         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6218                 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
6219                 ixl_vf_disable_queue_intr(hw, vfint_reg);
6220         }
6221
6222         /* Index 0 has a special register. */
6223         ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
6224
6225         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
6226                 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
6227                 ixl_vf_unregister_intr(hw, vpint_reg);
6228         }
6229
6230         vf->vsi.num_queues = 0;
6231 }
6232
6233 static int
6234 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
6235 {
6236         struct i40e_hw *hw;
6237         int i;
6238         uint16_t global_vf_num;
6239         uint32_t ciad;
6240
6241         hw = &pf->hw;
6242         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6243
6244         wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
6245              (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
6246         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6247                 ciad = rd32(hw, I40E_PF_PCI_CIAD);
6248                 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
6249                         return (0);
6250                 DELAY(1);
6251         }
6252
6253         return (ETIMEDOUT);
6254 }
6255
6256 static void
6257 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6258 {
6259         struct i40e_hw *hw;
6260         uint32_t vfrtrig;
6261
6262         hw = &pf->hw;
6263
6264         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6265         vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6266         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6267         ixl_flush(hw);
6268
6269         ixl_reinit_vf(pf, vf);
6270 }
6271
6272 static void
6273 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
6274 {
6275         struct i40e_hw *hw;
6276         uint32_t vfrstat, vfrtrig;
6277         int i, error;
6278
6279         hw = &pf->hw;
6280
6281         error = ixl_flush_pcie(pf, vf);
6282         if (error != 0)
6283                 device_printf(pf->dev,
6284                     "Timed out waiting for PCIe activity to stop on VF-%d\n",
6285                     vf->vf_num);
6286
6287         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
6288                 DELAY(10);
6289
6290                 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
6291                 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
6292                         break;
6293         }
6294
6295         if (i == IXL_VF_RESET_TIMEOUT)
6296                 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
6297
6298         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
6299
6300         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
6301         vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
6302         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
6303
6304         if (vf->vsi.seid != 0)
6305                 ixl_disable_rings(&vf->vsi);
6306
6307         ixl_vf_release_resources(pf, vf);
6308         ixl_vf_setup_vsi(pf, vf);
6309         ixl_vf_map_queues(pf, vf);
6310
6311         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
6312         ixl_flush(hw);
6313 }
6314
6315 static const char *
6316 ixl_vc_opcode_str(uint16_t op)
6317 {
6318
6319         switch (op) {
6320         case I40E_VIRTCHNL_OP_VERSION:
6321                 return ("VERSION");
6322         case I40E_VIRTCHNL_OP_RESET_VF:
6323                 return ("RESET_VF");
6324         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6325                 return ("GET_VF_RESOURCES");
6326         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6327                 return ("CONFIG_TX_QUEUE");
6328         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6329                 return ("CONFIG_RX_QUEUE");
6330         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6331                 return ("CONFIG_VSI_QUEUES");
6332         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6333                 return ("CONFIG_IRQ_MAP");
6334         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6335                 return ("ENABLE_QUEUES");
6336         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6337                 return ("DISABLE_QUEUES");
6338         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6339                 return ("ADD_ETHER_ADDRESS");
6340         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6341                 return ("DEL_ETHER_ADDRESS");
6342         case I40E_VIRTCHNL_OP_ADD_VLAN:
6343                 return ("ADD_VLAN");
6344         case I40E_VIRTCHNL_OP_DEL_VLAN:
6345                 return ("DEL_VLAN");
6346         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6347                 return ("CONFIG_PROMISCUOUS_MODE");
6348         case I40E_VIRTCHNL_OP_GET_STATS:
6349                 return ("GET_STATS");
6350         case I40E_VIRTCHNL_OP_FCOE:
6351                 return ("FCOE");
6352         case I40E_VIRTCHNL_OP_EVENT:
6353                 return ("EVENT");
6354         default:
6355                 return ("UNKNOWN");
6356         }
6357 }
6358
6359 static int
6360 ixl_vc_opcode_level(uint16_t opcode)
6361 {
6362         switch (opcode) {
6363         case I40E_VIRTCHNL_OP_GET_STATS:
6364                 return (10);
6365         default:
6366                 return (5);
6367         }
6368 }
6369
6370 static void
6371 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6372     enum i40e_status_code status, void *msg, uint16_t len)
6373 {
6374         struct i40e_hw *hw;
6375         int global_vf_id;
6376
6377         hw = &pf->hw;
6378         global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
6379
6380         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
6381             "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
6382             ixl_vc_opcode_str(op), op, status, vf->vf_num);
6383
6384         i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
6385 }
6386
6387 static void
6388 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
6389 {
6390
6391         ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
6392 }
6393
6394 static void
6395 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
6396     enum i40e_status_code status, const char *file, int line)
6397 {
6398
6399         I40E_VC_DEBUG(pf, 1,
6400             "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
6401             ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
6402         ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
6403 }
6404
6405 static void
6406 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6407     uint16_t msg_size)
6408 {
6409         struct i40e_virtchnl_version_info reply;
6410
6411         if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
6412                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
6413                     I40E_ERR_PARAM);
6414                 return;
6415         }
6416
6417         vf->version = ((struct i40e_virtchnl_version_info *)msg)->minor;
6418
6419         reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
6420         reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
6421         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
6422             sizeof(reply));
6423 }
6424
6425 static void
6426 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6427     uint16_t msg_size)
6428 {
6429
6430         if (msg_size != 0) {
6431                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
6432                     I40E_ERR_PARAM);
6433                 return;
6434         }
6435
6436         ixl_reset_vf(pf, vf);
6437
6438         /* No response to a reset message. */
6439 }
6440
6441 static void
6442 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6443     uint16_t msg_size)
6444 {
6445         struct i40e_virtchnl_vf_resource reply;
6446
6447         if ((vf->version == 0 && msg_size != 0) ||
6448             (vf->version == 1 && msg_size != 4)) {
6449                 device_printf(pf->dev, "Invalid GET_VF_RESOURCES message size,"
6450                     " for VF version %d.%d\n", I40E_VIRTCHNL_VERSION_MAJOR,
6451                     vf->version);
6452                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6453                     I40E_ERR_PARAM);
6454                 return;
6455         }
6456
6457         bzero(&reply, sizeof(reply));
6458
6459         if (vf->version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS)
6460                 reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
6461                                          I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
6462                                          I40E_VIRTCHNL_VF_OFFLOAD_VLAN;
6463         else
6464                 reply.vf_offload_flags = *(u32 *)msg;
6465
6466         reply.num_vsis = 1;
6467         reply.num_queue_pairs = vf->vsi.num_queues;
6468         reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
6469         reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
6470         reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
6471         reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
6472         memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
6473
6474         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
6475             I40E_SUCCESS, &reply, sizeof(reply));
6476 }
6477
6478 static int
6479 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6480     struct i40e_virtchnl_txq_info *info)
6481 {
6482         struct i40e_hw *hw;
6483         struct i40e_hmc_obj_txq txq;
6484         uint16_t global_queue_num, global_vf_num;
6485         enum i40e_status_code status;
6486         uint32_t qtx_ctl;
6487
6488         hw = &pf->hw;
6489         global_queue_num = vf->vsi.first_queue + info->queue_id;
6490         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
6491         bzero(&txq, sizeof(txq));
6492
6493         status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
6494         if (status != I40E_SUCCESS)
6495                 return (EINVAL);
6496
6497         txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
6498
6499         txq.head_wb_ena = info->headwb_enabled;
6500         txq.head_wb_addr = info->dma_headwb_addr;
6501         txq.qlen = info->ring_len;
6502         txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
6503         txq.rdylist_act = 0;
6504
6505         status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
6506         if (status != I40E_SUCCESS)
6507                 return (EINVAL);
6508
6509         qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
6510             (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
6511             (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
6512         wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
6513         ixl_flush(hw);
6514
6515         return (0);
6516 }
6517
6518 static int
6519 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
6520     struct i40e_virtchnl_rxq_info *info)
6521 {
6522         struct i40e_hw *hw;
6523         struct i40e_hmc_obj_rxq rxq;
6524         uint16_t global_queue_num;
6525         enum i40e_status_code status;
6526
6527         hw = &pf->hw;
6528         global_queue_num = vf->vsi.first_queue + info->queue_id;
6529         bzero(&rxq, sizeof(rxq));
6530
6531         if (info->databuffer_size > IXL_VF_MAX_BUFFER)
6532                 return (EINVAL);
6533
6534         if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
6535             info->max_pkt_size < ETHER_MIN_LEN)
6536                 return (EINVAL);
6537
6538         if (info->splithdr_enabled) {
6539                 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
6540                         return (EINVAL);
6541
6542                 rxq.hsplit_0 = info->rx_split_pos &
6543                     (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
6544                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
6545                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
6546                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
6547                 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
6548
6549                 rxq.dtype = 2;
6550         }
6551
6552         status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
6553         if (status != I40E_SUCCESS)
6554                 return (EINVAL);
6555
6556         rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
6557         rxq.qlen = info->ring_len;
6558
6559         rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
6560
6561         rxq.dsize = 1;
6562         rxq.crcstrip = 1;
6563         rxq.l2tsel = 1;
6564
6565         rxq.rxmax = info->max_pkt_size;
6566         rxq.tphrdesc_ena = 1;
6567         rxq.tphwdesc_ena = 1;
6568         rxq.tphdata_ena = 1;
6569         rxq.tphhead_ena = 1;
6570         rxq.lrxqthresh = 2;
6571         rxq.prefena = 1;
6572
6573         status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
6574         if (status != I40E_SUCCESS)
6575                 return (EINVAL);
6576
6577         return (0);
6578 }
6579
6580 static void
6581 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6582     uint16_t msg_size)
6583 {
6584         struct i40e_virtchnl_vsi_queue_config_info *info;
6585         struct i40e_virtchnl_queue_pair_info *pair;
6586         int i;
6587
6588         if (msg_size < sizeof(*info)) {
6589                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6590                     I40E_ERR_PARAM);
6591                 return;
6592         }
6593
6594         info = msg;
6595         if (info->num_queue_pairs == 0) {
6596                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6597                     I40E_ERR_PARAM);
6598                 return;
6599         }
6600
6601         if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
6602                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6603                     I40E_ERR_PARAM);
6604                 return;
6605         }
6606
6607         if (info->vsi_id != vf->vsi.vsi_num) {
6608                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
6609                     I40E_ERR_PARAM);
6610                 return;
6611         }
6612
6613         for (i = 0; i < info->num_queue_pairs; i++) {
6614                 pair = &info->qpair[i];
6615
6616                 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
6617                     pair->rxq.vsi_id != vf->vsi.vsi_num ||
6618                     pair->txq.queue_id != pair->rxq.queue_id ||
6619                     pair->txq.queue_id >= vf->vsi.num_queues) {
6620
6621                         i40e_send_vf_nack(pf, vf,
6622                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6623                         return;
6624                 }
6625
6626                 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
6627                         i40e_send_vf_nack(pf, vf,
6628                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6629                         return;
6630                 }
6631
6632                 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
6633                         i40e_send_vf_nack(pf, vf,
6634                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
6635                         return;
6636                 }
6637         }
6638
6639         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
6640 }
6641
6642 static void
6643 ixl_vf_set_qctl(struct ixl_pf *pf,
6644     const struct i40e_virtchnl_vector_map *vector,
6645     enum i40e_queue_type cur_type, uint16_t cur_queue,
6646     enum i40e_queue_type *last_type, uint16_t *last_queue)
6647 {
6648         uint32_t offset, qctl;
6649         uint16_t itr_indx;
6650
6651         if (cur_type == I40E_QUEUE_TYPE_RX) {
6652                 offset = I40E_QINT_RQCTL(cur_queue);
6653                 itr_indx = vector->rxitr_idx;
6654         } else {
6655                 offset = I40E_QINT_TQCTL(cur_queue);
6656                 itr_indx = vector->txitr_idx;
6657         }
6658
6659         qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
6660             (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
6661             (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
6662             I40E_QINT_RQCTL_CAUSE_ENA_MASK |
6663             (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
6664
6665         wr32(&pf->hw, offset, qctl);
6666
6667         *last_type = cur_type;
6668         *last_queue = cur_queue;
6669 }
6670
6671 static void
6672 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
6673     const struct i40e_virtchnl_vector_map *vector)
6674 {
6675         struct i40e_hw *hw;
6676         u_int qindex;
6677         enum i40e_queue_type type, last_type;
6678         uint32_t lnklst_reg;
6679         uint16_t rxq_map, txq_map, cur_queue, last_queue;
6680
6681         hw = &pf->hw;
6682
6683         rxq_map = vector->rxq_map;
6684         txq_map = vector->txq_map;
6685
6686         last_queue = IXL_END_OF_INTR_LNKLST;
6687         last_type = I40E_QUEUE_TYPE_RX;
6688
6689         /*
6690          * The datasheet says to optimize performance, RX queues and TX queues
6691          * should be interleaved in the interrupt linked list, so we process
6692          * both at once here.
6693          */
6694         while ((rxq_map != 0) || (txq_map != 0)) {
6695                 if (txq_map != 0) {
6696                         qindex = ffs(txq_map) - 1;
6697                         type = I40E_QUEUE_TYPE_TX;
6698                         cur_queue = vf->vsi.first_queue + qindex;
6699                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
6700                             &last_type, &last_queue);
6701                         txq_map &= ~(1 << qindex);
6702                 }
6703
6704                 if (rxq_map != 0) {
6705                         qindex = ffs(rxq_map) - 1;
6706                         type = I40E_QUEUE_TYPE_RX;
6707                         cur_queue = vf->vsi.first_queue + qindex;
6708                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
6709                             &last_type, &last_queue);
6710                         rxq_map &= ~(1 << qindex);
6711                 }
6712         }
6713
6714         if (vector->vector_id == 0)
6715                 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
6716         else
6717                 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
6718                     vf->vf_num);
6719         wr32(hw, lnklst_reg,
6720             (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
6721             (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
6722
6723         ixl_flush(hw);
6724 }
6725
6726 static void
6727 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6728     uint16_t msg_size)
6729 {
6730         struct i40e_virtchnl_irq_map_info *map;
6731         struct i40e_virtchnl_vector_map *vector;
6732         struct i40e_hw *hw;
6733         int i, largest_txq, largest_rxq;
6734
6735         hw = &pf->hw;
6736
6737         if (msg_size < sizeof(*map)) {
6738                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6739                     I40E_ERR_PARAM);
6740                 return;
6741         }
6742
6743         map = msg;
6744         if (map->num_vectors == 0) {
6745                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6746                     I40E_ERR_PARAM);
6747                 return;
6748         }
6749
6750         if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
6751                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6752                     I40E_ERR_PARAM);
6753                 return;
6754         }
6755
6756         for (i = 0; i < map->num_vectors; i++) {
6757                 vector = &map->vecmap[i];
6758
6759                 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
6760                     vector->vsi_id != vf->vsi.vsi_num) {
6761                         i40e_send_vf_nack(pf, vf,
6762                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
6763                         return;
6764                 }
6765
6766                 if (vector->rxq_map != 0) {
6767                         largest_rxq = fls(vector->rxq_map) - 1;
6768                         if (largest_rxq >= vf->vsi.num_queues) {
6769                                 i40e_send_vf_nack(pf, vf,
6770                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6771                                     I40E_ERR_PARAM);
6772                                 return;
6773                         }
6774                 }
6775
6776                 if (vector->txq_map != 0) {
6777                         largest_txq = fls(vector->txq_map) - 1;
6778                         if (largest_txq >= vf->vsi.num_queues) {
6779                                 i40e_send_vf_nack(pf, vf,
6780                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6781                                     I40E_ERR_PARAM);
6782                                 return;
6783                         }
6784                 }
6785
6786                 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6787                     vector->txitr_idx > IXL_MAX_ITR_IDX) {
6788                         i40e_send_vf_nack(pf, vf,
6789                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6790                             I40E_ERR_PARAM);
6791                         return;
6792                 }
6793
6794                 ixl_vf_config_vector(pf, vf, vector);
6795         }
6796
6797         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6798 }
6799
6800 static void
6801 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6802     uint16_t msg_size)
6803 {
6804         struct i40e_virtchnl_queue_select *select;
6805         int error;
6806
6807         if (msg_size != sizeof(*select)) {
6808                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6809                     I40E_ERR_PARAM);
6810                 return;
6811         }
6812
6813         select = msg;
6814         if (select->vsi_id != vf->vsi.vsi_num ||
6815             select->rx_queues == 0 || select->tx_queues == 0) {
6816                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6817                     I40E_ERR_PARAM);
6818                 return;
6819         }
6820
6821         error = ixl_enable_rings(&vf->vsi);
6822         if (error) {
6823                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6824                     I40E_ERR_TIMEOUT);
6825                 return;
6826         }
6827
6828         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6829 }
6830
6831 static void
6832 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6833     void *msg, uint16_t msg_size)
6834 {
6835         struct i40e_virtchnl_queue_select *select;
6836         int error;
6837
6838         if (msg_size != sizeof(*select)) {
6839                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6840                     I40E_ERR_PARAM);
6841                 return;
6842         }
6843
6844         select = msg;
6845         if (select->vsi_id != vf->vsi.vsi_num ||
6846             select->rx_queues == 0 || select->tx_queues == 0) {
6847                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6848                     I40E_ERR_PARAM);
6849                 return;
6850         }
6851
6852         error = ixl_disable_rings(&vf->vsi);
6853         if (error) {
6854                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6855                     I40E_ERR_TIMEOUT);
6856                 return;
6857         }
6858
6859         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6860 }
6861
6862 static boolean_t
6863 ixl_zero_mac(const uint8_t *addr)
6864 {
6865         uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6866
6867         return (cmp_etheraddr(addr, zero));
6868 }
6869
6870 static boolean_t
6871 ixl_bcast_mac(const uint8_t *addr)
6872 {
6873
6874         return (cmp_etheraddr(addr, ixl_bcast_addr));
6875 }
6876
6877 static int
6878 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6879 {
6880
6881         if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6882                 return (EINVAL);
6883
6884         /*
6885          * If the VF is not allowed to change its MAC address, don't let it
6886          * set a MAC filter for an address that is not a multicast address and
6887          * is not its assigned MAC.
6888          */
6889         if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6890             !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6891                 return (EPERM);
6892
6893         return (0);
6894 }
6895
6896 static void
6897 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6898     uint16_t msg_size)
6899 {
6900         struct i40e_virtchnl_ether_addr_list *addr_list;
6901         struct i40e_virtchnl_ether_addr *addr;
6902         struct ixl_vsi *vsi;
6903         int i;
6904         size_t expected_size;
6905
6906         vsi = &vf->vsi;
6907
6908         if (msg_size < sizeof(*addr_list)) {
6909                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6910                     I40E_ERR_PARAM);
6911                 return;
6912         }
6913
6914         addr_list = msg;
6915         expected_size = sizeof(*addr_list) +
6916             addr_list->num_elements * sizeof(*addr);
6917
6918         if (addr_list->num_elements == 0 ||
6919             addr_list->vsi_id != vsi->vsi_num ||
6920             msg_size != expected_size) {
6921                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6922                     I40E_ERR_PARAM);
6923                 return;
6924         }
6925
6926         for (i = 0; i < addr_list->num_elements; i++) {
6927                 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6928                         i40e_send_vf_nack(pf, vf,
6929                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6930                         return;
6931                 }
6932         }
6933
6934         for (i = 0; i < addr_list->num_elements; i++) {
6935                 addr = &addr_list->list[i];
6936                 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6937         }
6938
6939         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6940 }
6941
6942 static void
6943 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6944     uint16_t msg_size)
6945 {
6946         struct i40e_virtchnl_ether_addr_list *addr_list;
6947         struct i40e_virtchnl_ether_addr *addr;
6948         size_t expected_size;
6949         int i;
6950
6951         if (msg_size < sizeof(*addr_list)) {
6952                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6953                     I40E_ERR_PARAM);
6954                 return;
6955         }
6956
6957         addr_list = msg;
6958         expected_size = sizeof(*addr_list) +
6959             addr_list->num_elements * sizeof(*addr);
6960
6961         if (addr_list->num_elements == 0 ||
6962             addr_list->vsi_id != vf->vsi.vsi_num ||
6963             msg_size != expected_size) {
6964                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6965                     I40E_ERR_PARAM);
6966                 return;
6967         }
6968
6969         for (i = 0; i < addr_list->num_elements; i++) {
6970                 addr = &addr_list->list[i];
6971                 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6972                         i40e_send_vf_nack(pf, vf,
6973                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6974                         return;
6975                 }
6976         }
6977
6978         for (i = 0; i < addr_list->num_elements; i++) {
6979                 addr = &addr_list->list[i];
6980                 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6981         }
6982
6983         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6984 }
6985
6986 static enum i40e_status_code
6987 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6988 {
6989         struct i40e_vsi_context vsi_ctx;
6990
6991         vsi_ctx.seid = vf->vsi.seid;
6992
6993         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6994         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6995         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6996             I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6997         return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6998 }
6999
7000 static void
7001 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7002     uint16_t msg_size)
7003 {
7004         struct i40e_virtchnl_vlan_filter_list *filter_list;
7005         enum i40e_status_code code;
7006         size_t expected_size;
7007         int i;
7008
7009         if (msg_size < sizeof(*filter_list)) {
7010                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7011                     I40E_ERR_PARAM);
7012                 return;
7013         }
7014
7015         filter_list = msg;
7016         expected_size = sizeof(*filter_list) +
7017             filter_list->num_elements * sizeof(uint16_t);
7018         if (filter_list->num_elements == 0 ||
7019             filter_list->vsi_id != vf->vsi.vsi_num ||
7020             msg_size != expected_size) {
7021                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7022                     I40E_ERR_PARAM);
7023                 return;
7024         }
7025
7026         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7027                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7028                     I40E_ERR_PARAM);
7029                 return;
7030         }
7031
7032         for (i = 0; i < filter_list->num_elements; i++) {
7033                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7034                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7035                             I40E_ERR_PARAM);
7036                         return;
7037                 }
7038         }
7039
7040         code = ixl_vf_enable_vlan_strip(pf, vf);
7041         if (code != I40E_SUCCESS) {
7042                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7043                     I40E_ERR_PARAM);
7044         }
7045
7046         for (i = 0; i < filter_list->num_elements; i++)
7047                 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7048
7049         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
7050 }
7051
7052 static void
7053 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7054     uint16_t msg_size)
7055 {
7056         struct i40e_virtchnl_vlan_filter_list *filter_list;
7057         int i;
7058         size_t expected_size;
7059
7060         if (msg_size < sizeof(*filter_list)) {
7061                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7062                     I40E_ERR_PARAM);
7063                 return;
7064         }
7065
7066         filter_list = msg;
7067         expected_size = sizeof(*filter_list) +
7068             filter_list->num_elements * sizeof(uint16_t);
7069         if (filter_list->num_elements == 0 ||
7070             filter_list->vsi_id != vf->vsi.vsi_num ||
7071             msg_size != expected_size) {
7072                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
7073                     I40E_ERR_PARAM);
7074                 return;
7075         }
7076
7077         for (i = 0; i < filter_list->num_elements; i++) {
7078                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
7079                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7080                             I40E_ERR_PARAM);
7081                         return;
7082                 }
7083         }
7084
7085         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
7086                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
7087                     I40E_ERR_PARAM);
7088                 return;
7089         }
7090
7091         for (i = 0; i < filter_list->num_elements; i++)
7092                 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
7093
7094         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
7095 }
7096
7097 static void
7098 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
7099     void *msg, uint16_t msg_size)
7100 {
7101         struct i40e_virtchnl_promisc_info *info;
7102         enum i40e_status_code code;
7103
7104         if (msg_size != sizeof(*info)) {
7105                 i40e_send_vf_nack(pf, vf,
7106                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7107                 return;
7108         }
7109
7110         if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) {
7111                 i40e_send_vf_nack(pf, vf,
7112                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7113                 return;
7114         }
7115
7116         info = msg;
7117         if (info->vsi_id != vf->vsi.vsi_num) {
7118                 i40e_send_vf_nack(pf, vf,
7119                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
7120                 return;
7121         }
7122
7123         code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
7124             info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
7125         if (code != I40E_SUCCESS) {
7126                 i40e_send_vf_nack(pf, vf,
7127                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7128                 return;
7129         }
7130
7131         code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
7132             info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
7133         if (code != I40E_SUCCESS) {
7134                 i40e_send_vf_nack(pf, vf,
7135                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
7136                 return;
7137         }
7138
7139         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
7140 }
7141
7142 static void
7143 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
7144     uint16_t msg_size)
7145 {
7146         struct i40e_virtchnl_queue_select *queue;
7147
7148         if (msg_size != sizeof(*queue)) {
7149                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7150                     I40E_ERR_PARAM);
7151                 return;
7152         }
7153
7154         queue = msg;
7155         if (queue->vsi_id != vf->vsi.vsi_num) {
7156                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7157                     I40E_ERR_PARAM);
7158                 return;
7159         }
7160
7161         ixl_update_eth_stats(&vf->vsi);
7162
7163         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
7164             I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
7165 }
7166
7167 static void
7168 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
7169 {
7170         struct ixl_vf *vf;
7171         void *msg;
7172         uint16_t vf_num, msg_size;
7173         uint32_t opcode;
7174
7175         vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
7176         opcode = le32toh(event->desc.cookie_high);
7177
7178         if (vf_num >= pf->num_vfs) {
7179                 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
7180                 return;
7181         }
7182
7183         vf = &pf->vfs[vf_num];
7184         msg = event->msg_buf;
7185         msg_size = event->msg_len;
7186
7187         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
7188             "Got msg %s(%d) from VF-%d of size %d\n",
7189             ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
7190
7191         switch (opcode) {
7192         case I40E_VIRTCHNL_OP_VERSION:
7193                 ixl_vf_version_msg(pf, vf, msg, msg_size);
7194                 break;
7195         case I40E_VIRTCHNL_OP_RESET_VF:
7196                 ixl_vf_reset_msg(pf, vf, msg, msg_size);
7197                 break;
7198         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
7199                 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
7200                 break;
7201         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
7202                 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
7203                 break;
7204         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
7205                 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
7206                 break;
7207         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
7208                 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
7209                 break;
7210         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
7211                 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
7212                 break;
7213         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
7214                 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
7215                 break;
7216         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
7217                 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
7218                 break;
7219         case I40E_VIRTCHNL_OP_ADD_VLAN:
7220                 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
7221                 break;
7222         case I40E_VIRTCHNL_OP_DEL_VLAN:
7223                 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
7224                 break;
7225         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
7226                 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
7227                 break;
7228         case I40E_VIRTCHNL_OP_GET_STATS:
7229                 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
7230                 break;
7231
7232         /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
7233         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
7234         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
7235         default:
7236                 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
7237                 break;
7238         }
7239 }
7240
7241 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
7242 static void
7243 ixl_handle_vflr(void *arg, int pending)
7244 {
7245         struct ixl_pf *pf;
7246         struct i40e_hw *hw;
7247         uint16_t global_vf_num;
7248         uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
7249         int i;
7250
7251         pf = arg;
7252         hw = &pf->hw;
7253
7254         IXL_PF_LOCK(pf);
7255         for (i = 0; i < pf->num_vfs; i++) {
7256                 global_vf_num = hw->func_caps.vf_base_id + i;
7257
7258                 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
7259                 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
7260                 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
7261                 if (vflrstat & vflrstat_mask) {
7262                         wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
7263                             vflrstat_mask);
7264
7265                         ixl_reinit_vf(pf, &pf->vfs[i]);
7266                 }
7267         }
7268
7269         icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
7270         icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
7271         wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
7272         ixl_flush(hw);
7273
7274         IXL_PF_UNLOCK(pf);
7275 }
7276
7277 static int
7278 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
7279 {
7280
7281         switch (err) {
7282         case I40E_AQ_RC_EPERM:
7283                 return (EPERM);
7284         case I40E_AQ_RC_ENOENT:
7285                 return (ENOENT);
7286         case I40E_AQ_RC_ESRCH:
7287                 return (ESRCH);
7288         case I40E_AQ_RC_EINTR:
7289                 return (EINTR);
7290         case I40E_AQ_RC_EIO:
7291                 return (EIO);
7292         case I40E_AQ_RC_ENXIO:
7293                 return (ENXIO);
7294         case I40E_AQ_RC_E2BIG:
7295                 return (E2BIG);
7296         case I40E_AQ_RC_EAGAIN:
7297                 return (EAGAIN);
7298         case I40E_AQ_RC_ENOMEM:
7299                 return (ENOMEM);
7300         case I40E_AQ_RC_EACCES:
7301                 return (EACCES);
7302         case I40E_AQ_RC_EFAULT:
7303                 return (EFAULT);
7304         case I40E_AQ_RC_EBUSY:
7305                 return (EBUSY);
7306         case I40E_AQ_RC_EEXIST:
7307                 return (EEXIST);
7308         case I40E_AQ_RC_EINVAL:
7309                 return (EINVAL);
7310         case I40E_AQ_RC_ENOTTY:
7311                 return (ENOTTY);
7312         case I40E_AQ_RC_ENOSPC:
7313                 return (ENOSPC);
7314         case I40E_AQ_RC_ENOSYS:
7315                 return (ENOSYS);
7316         case I40E_AQ_RC_ERANGE:
7317                 return (ERANGE);
7318         case I40E_AQ_RC_EFLUSHED:
7319                 return (EINVAL);        /* No exact equivalent in errno.h */
7320         case I40E_AQ_RC_BAD_ADDR:
7321                 return (EFAULT);
7322         case I40E_AQ_RC_EMODE:
7323                 return (EPERM);
7324         case I40E_AQ_RC_EFBIG:
7325                 return (EFBIG);
7326         default:
7327                 return (EINVAL);
7328         }
7329 }
7330
7331 static int
7332 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
7333 {
7334         struct ixl_pf *pf;
7335         struct i40e_hw *hw;
7336         struct ixl_vsi *pf_vsi;
7337         enum i40e_status_code ret;
7338         int i, error;
7339
7340         pf = device_get_softc(dev);
7341         hw = &pf->hw;
7342         pf_vsi = &pf->vsi;
7343
7344         IXL_PF_LOCK(pf);
7345         pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
7346             M_ZERO);
7347
7348         if (pf->vfs == NULL) {
7349                 error = ENOMEM;
7350                 goto fail;
7351         }
7352
7353         for (i = 0; i < num_vfs; i++)
7354                 sysctl_ctx_init(&pf->vfs[i].ctx);
7355
7356         ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
7357             1, FALSE, &pf->veb_seid, FALSE, NULL);
7358         if (ret != I40E_SUCCESS) {
7359                 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
7360                 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
7361                     error);
7362                 goto fail;
7363         }
7364
7365         // TODO: [Configure MSI-X here]
7366         ixl_enable_adminq(hw);
7367
7368         pf->num_vfs = num_vfs;
7369         IXL_PF_UNLOCK(pf);
7370         return (0);
7371
7372 fail:
7373         free(pf->vfs, M_IXL);
7374         pf->vfs = NULL;
7375         IXL_PF_UNLOCK(pf);
7376         return (error);
7377 }
7378
7379 static void
7380 ixl_iov_uninit(device_t dev)
7381 {
7382         struct ixl_pf *pf;
7383         struct i40e_hw *hw;
7384         struct ixl_vsi *vsi;
7385         struct ifnet *ifp;
7386         struct ixl_vf *vfs;
7387         int i, num_vfs;
7388
7389         pf = device_get_softc(dev);
7390         hw = &pf->hw;
7391         vsi = &pf->vsi;
7392         ifp = vsi->ifp;
7393
7394         IXL_PF_LOCK(pf);
7395         for (i = 0; i < pf->num_vfs; i++) {
7396                 if (pf->vfs[i].vsi.seid != 0)
7397                         i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
7398         }
7399
7400         if (pf->veb_seid != 0) {
7401                 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
7402                 pf->veb_seid = 0;
7403         }
7404
7405         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
7406                 ixl_disable_intr(vsi);
7407
7408         vfs = pf->vfs;
7409         num_vfs = pf->num_vfs;
7410
7411         pf->vfs = NULL;
7412         pf->num_vfs = 0;
7413         IXL_PF_UNLOCK(pf);
7414
7415         /* Do this after the unlock as sysctl_ctx_free might sleep. */
7416         for (i = 0; i < num_vfs; i++)
7417                 sysctl_ctx_free(&vfs[i].ctx);
7418         free(vfs, M_IXL);
7419 }
7420
7421 static int
7422 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
7423 {
7424         char sysctl_name[QUEUE_NAME_LEN];
7425         struct ixl_pf *pf;
7426         struct ixl_vf *vf;
7427         const void *mac;
7428         size_t size;
7429         int error;
7430
7431         pf = device_get_softc(dev);
7432         vf = &pf->vfs[vfnum];
7433
7434         IXL_PF_LOCK(pf);
7435         vf->vf_num = vfnum;
7436
7437         vf->vsi.back = pf;
7438         vf->vf_flags = VF_FLAG_ENABLED;
7439         SLIST_INIT(&vf->vsi.ftl);
7440
7441         error = ixl_vf_setup_vsi(pf, vf);
7442         if (error != 0)
7443                 goto out;
7444
7445         if (nvlist_exists_binary(params, "mac-addr")) {
7446                 mac = nvlist_get_binary(params, "mac-addr", &size);
7447                 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
7448
7449                 if (nvlist_get_bool(params, "allow-set-mac"))
7450                         vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7451         } else
7452                 /*
7453                  * If the administrator has not specified a MAC address then
7454                  * we must allow the VF to choose one.
7455                  */
7456                 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
7457
7458         if (nvlist_get_bool(params, "mac-anti-spoof"))
7459                 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
7460
7461         if (nvlist_get_bool(params, "allow-promisc"))
7462                 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
7463
7464         /* TODO: Get VLAN that PF has set for the VF */
7465
7466         vf->vf_flags |= VF_FLAG_VLAN_CAP;
7467
7468         ixl_reset_vf(pf, vf);
7469 out:
7470         IXL_PF_UNLOCK(pf);
7471         if (error == 0) {
7472                 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
7473                 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
7474         }
7475
7476         return (error);
7477 }
7478 #endif /* PCI_IOV */