sys/dev/ixl/if_ixl.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifndef IXL_STANDALONE_BUILD
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #endif
  39
  40 #include "ixl.h"
  41 #include "ixl_pf.h"
  42
  43 #ifdef RSS
  44 #include <net/rss_config.h>
  45 #endif
  46
  47 /*********************************************************************
  48  *  Driver version
  49  *********************************************************************/
  50 char ixl_driver_version[] = "1.4.1";
  51
  52 /*********************************************************************
  53  *  PCI Device ID Table
  54  *
  55  *  Used by probe to select devices to load on
  56  *  Last field stores an index into ixl_strings
  57  *  Last entry must be all 0s
  58  *
  59  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  60  *********************************************************************/
  61
  62 static ixl_vendor_info_t ixl_vendor_info_array[] =
  63 {
  64         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
  65         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_A, 0, 0, 0},
  66         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
  67         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
  68         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
  69         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
  70         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
  71         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
  72         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2, 0, 0, 0},
  73         /* required last entry */
  74         {0, 0, 0, 0, 0}
  75 };
  76
  77 /*********************************************************************
  78  *  Table of branding strings
  79  *********************************************************************/
  80
  81 static char    *ixl_strings[] = {
  82         "Intel(R) Ethernet Connection XL710 Driver"
  83 };
  84
  85
  86 /*********************************************************************
  87  *  Function prototypes
  88  *********************************************************************/
  89 static int      ixl_probe(device_t);
  90 static int      ixl_attach(device_t);
  91 static int      ixl_detach(device_t);
  92 static int      ixl_shutdown(device_t);
  93 static int      ixl_get_hw_capabilities(struct ixl_pf *);
  94 static void     ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
  95 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
  96 static void     ixl_init(void *);
  97 static void     ixl_init_locked(struct ixl_pf *);
  98 static void     ixl_stop(struct ixl_pf *);
  99 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
 100 static int      ixl_media_change(struct ifnet *);
 101 static void     ixl_update_link_status(struct ixl_pf *);
 102 static int      ixl_allocate_pci_resources(struct ixl_pf *);
 103 static u16      ixl_get_bus_info(struct i40e_hw *, device_t);
 104 static int      ixl_setup_stations(struct ixl_pf *);
 105 static int      ixl_switch_config(struct ixl_pf *);
 106 static int      ixl_initialize_vsi(struct ixl_vsi *);
 107 static int      ixl_assign_vsi_msix(struct ixl_pf *);
 108 static int      ixl_assign_vsi_legacy(struct ixl_pf *);
 109 static int      ixl_init_msix(struct ixl_pf *);
 110 static void     ixl_configure_msix(struct ixl_pf *);
 111 static void     ixl_configure_itr(struct ixl_pf *);
 112 static void     ixl_configure_legacy(struct ixl_pf *);
 113 static void     ixl_free_pci_resources(struct ixl_pf *);
 114 static void     ixl_local_timer(void *);
 115 static int      ixl_setup_interface(device_t, struct ixl_vsi *);
 116 static void     ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
 117 static void     ixl_config_rss(struct ixl_vsi *);
 118 static void     ixl_set_queue_rx_itr(struct ixl_queue *);
 119 static void     ixl_set_queue_tx_itr(struct ixl_queue *);
 120 static int      ixl_set_advertised_speeds(struct ixl_pf *, int);
 121
 122 static int      ixl_enable_rings(struct ixl_vsi *);
 123 static int      ixl_disable_rings(struct ixl_vsi *);
 124 static void     ixl_enable_intr(struct ixl_vsi *);
 125 static void     ixl_disable_intr(struct ixl_vsi *);
 126 static void     ixl_disable_rings_intr(struct ixl_vsi *);
 127
 128 static void     ixl_enable_adminq(struct i40e_hw *);
 129 static void     ixl_disable_adminq(struct i40e_hw *);
 130 static void     ixl_enable_queue(struct i40e_hw *, int);
 131 static void     ixl_disable_queue(struct i40e_hw *, int);
 132 static void     ixl_enable_legacy(struct i40e_hw *);
 133 static void     ixl_disable_legacy(struct i40e_hw *);
 134
 135 static void     ixl_set_promisc(struct ixl_vsi *);
 136 static void     ixl_add_multi(struct ixl_vsi *);
 137 static void     ixl_del_multi(struct ixl_vsi *);
 138 static void     ixl_register_vlan(void *, struct ifnet *, u16);
 139 static void     ixl_unregister_vlan(void *, struct ifnet *, u16);
 140 static void     ixl_setup_vlan_filters(struct ixl_vsi *);
 141
 142 static void     ixl_init_filters(struct ixl_vsi *);
 143 static void     ixl_reconfigure_filters(struct ixl_vsi *vsi);
 144 static void     ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
 145 static void     ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
 146 static void     ixl_add_hw_filters(struct ixl_vsi *, int, int);
 147 static void     ixl_del_hw_filters(struct ixl_vsi *, int);
 148 static struct ixl_mac_filter *
 149                 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
 150 static void     ixl_add_mc_filter(struct ixl_vsi *, u8 *);
 151 static void     ixl_free_mac_filters(struct ixl_vsi *vsi);
 152
 153
 154 /* Sysctl debug interface */
 155 static int      ixl_debug_info(SYSCTL_HANDLER_ARGS);
 156 static void     ixl_print_debug_info(struct ixl_pf *);
 157
 158 /* The MSI/X Interrupt handlers */
 159 static void     ixl_intr(void *);
 160 static void     ixl_msix_que(void *);
 161 static void     ixl_msix_adminq(void *);
 162 static void     ixl_handle_mdd_event(struct ixl_pf *);
 163
 164 /* Deferred interrupt tasklets */
 165 static void     ixl_do_adminq(void *, int);
 166
 167 /* Sysctl handlers */
 168 static int      ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
 169 static int      ixl_set_advertise(SYSCTL_HANDLER_ARGS);
 170 static int      ixl_current_speed(SYSCTL_HANDLER_ARGS);
 171 static int      ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
 172
 173 /* Statistics */
 174 static void     ixl_add_hw_stats(struct ixl_pf *);
 175 static void     ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
 176                     struct sysctl_oid_list *, struct i40e_hw_port_stats *);
 177 static void     ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
 178                     struct sysctl_oid_list *,
 179                     struct i40e_eth_stats *);
 180 static void     ixl_update_stats_counters(struct ixl_pf *);
 181 static void     ixl_update_eth_stats(struct ixl_vsi *);
 182 static void     ixl_update_vsi_stats(struct ixl_vsi *);
 183 static void     ixl_pf_reset_stats(struct ixl_pf *);
 184 static void     ixl_vsi_reset_stats(struct ixl_vsi *);
 185 static void     ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
 186                     u64 *, u64 *);
 187 static void     ixl_stat_update32(struct i40e_hw *, u32, bool,
 188                     u64 *, u64 *);
 189
 190 #ifdef IXL_DEBUG_SYSCTL
 191 static int      ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
 192 static int      ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
 193 static int      ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 194 static int      ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
 195 static int      ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
 196 #endif
 197
 198 #ifdef PCI_IOV
 199 static int      ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
 200
 201 static int      ixl_init_iov(device_t dev, uint16_t num_vfs, const nvlist_t*);
 202 static void     ixl_uninit_iov(device_t dev);
 203 static int      ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
 204
 205 static void     ixl_handle_vf_msg(struct ixl_pf *,
 206                     struct i40e_arq_event_info *);
 207 static void     ixl_handle_vflr(void *arg, int pending);
 208
 209 static void     ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 210 static void     ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 211 #endif
 212
 213 /*********************************************************************
 214  *  FreeBSD Device Interface Entry Points
 215  *********************************************************************/
 216
 217 static device_method_t ixl_methods[] = {
 218         /* Device interface */
 219         DEVMETHOD(device_probe, ixl_probe),
 220         DEVMETHOD(device_attach, ixl_attach),
 221         DEVMETHOD(device_detach, ixl_detach),
 222         DEVMETHOD(device_shutdown, ixl_shutdown),
 223 #ifdef PCI_IOV
 224         DEVMETHOD(pci_init_iov, ixl_init_iov),
 225         DEVMETHOD(pci_uninit_iov, ixl_uninit_iov),
 226         DEVMETHOD(pci_add_vf, ixl_add_vf),
 227 #endif
 228         {0, 0}
 229 };
 230
 231 static driver_t ixl_driver = {
 232         "ixl", ixl_methods, sizeof(struct ixl_pf),
 233 };
 234
 235 devclass_t ixl_devclass;
 236 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
 237
 238 MODULE_DEPEND(ixl, pci, 1, 1, 1);
 239 MODULE_DEPEND(ixl, ether, 1, 1, 1);
 240
 241 #ifdef DEV_NETMAP
 242 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
 243 #endif /* DEV_NETMAP */
 244
 245
 246 /*
 247 ** Global reset mutex
 248 */
 249 static struct mtx ixl_reset_mtx;
 250
 251 /*
 252 ** TUNEABLE PARAMETERS:
 253 */
 254
 255 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
 256                    "IXL driver parameters");
 257
 258 /*
 259  * MSIX should be the default for best performance,
 260  * but this allows it to be forced off for testing.
 261  */
 262 static int ixl_enable_msix = 1;
 263 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
 264 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
 265     "Enable MSI-X interrupts");
 266
 267 /*
 268 ** Number of descriptors per ring:
 269 **   - TX and RX are the same size
 270 */
 271 static int ixl_ringsz = DEFAULT_RING;
 272 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
 273 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
 274     &ixl_ringsz, 0, "Descriptor Ring Size");
 275
 276 /*
 277 ** This can be set manually, if left as 0 the
 278 ** number of queues will be calculated based
 279 ** on cpus and msix vectors available.
 280 */
 281 int ixl_max_queues = 0;
 282 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
 283 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
 284     &ixl_max_queues, 0, "Number of Queues");
 285
 286 /*
 287 ** Controls for Interrupt Throttling
 288 **      - true/false for dynamic adjustment
 289 **      - default values for static ITR
 290 */
 291 int ixl_dynamic_rx_itr = 0;
 292 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
 293 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
 294     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
 295
 296 int ixl_dynamic_tx_itr = 0;
 297 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
 298 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
 299     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
 300
 301 int ixl_rx_itr = IXL_ITR_8K;
 302 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
 303 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
 304     &ixl_rx_itr, 0, "RX Interrupt Rate");
 305
 306 int ixl_tx_itr = IXL_ITR_4K;
 307 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
 308 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
 309     &ixl_tx_itr, 0, "TX Interrupt Rate");
 310
 311 #ifdef IXL_FDIR
 312 static int ixl_enable_fdir = 1;
 313 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
 314 /* Rate at which we sample */
 315 int ixl_atr_rate = 20;
 316 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
 317 #endif
 318
 319 #ifdef DEV_NETMAP
 320 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
 321 #include <dev/netmap/if_ixl_netmap.h>
 322 #endif /* DEV_NETMAP */
 323
 324 static char *ixl_fc_string[6] = {
 325         "None",
 326         "Rx",
 327         "Tx",
 328         "Full",
 329         "Priority",
 330         "Default"
 331 };
 332
 333 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
 334
 335 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
 336     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 337
 338 /*********************************************************************
 339  *  Device identification routine
 340  *
 341  *  ixl_probe determines if the driver should be loaded on
 342  *  the hardware based on PCI vendor/device id of the device.
 343  *
 344  *  return BUS_PROBE_DEFAULT on success, positive on failure
 345  *********************************************************************/
 346
 347 static int
 348 ixl_probe(device_t dev)
 349 {
 350         ixl_vendor_info_t *ent;
 351
 352         u16     pci_vendor_id, pci_device_id;
 353         u16     pci_subvendor_id, pci_subdevice_id;
 354         char    device_name[256];
 355         static bool lock_init = FALSE;
 356
 357         INIT_DEBUGOUT("ixl_probe: begin");
 358
 359         pci_vendor_id = pci_get_vendor(dev);
 360         if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
 361                 return (ENXIO);
 362
 363         pci_device_id = pci_get_device(dev);
 364         pci_subvendor_id = pci_get_subvendor(dev);
 365         pci_subdevice_id = pci_get_subdevice(dev);
 366
 367         ent = ixl_vendor_info_array;
 368         while (ent->vendor_id != 0) {
 369                 if ((pci_vendor_id == ent->vendor_id) &&
 370                     (pci_device_id == ent->device_id) &&
 371
 372                     ((pci_subvendor_id == ent->subvendor_id) ||
 373                      (ent->subvendor_id == 0)) &&
 374
 375                     ((pci_subdevice_id == ent->subdevice_id) ||
 376                      (ent->subdevice_id == 0))) {
 377                         sprintf(device_name, "%s, Version - %s",
 378                                 ixl_strings[ent->index],
 379                                 ixl_driver_version);
 380                         device_set_desc_copy(dev, device_name);
 381                         /* One shot mutex init */
 382                         if (lock_init == FALSE) {
 383                                 lock_init = TRUE;
 384                                 mtx_init(&ixl_reset_mtx,
 385                                     "ixl_reset",
 386                                     "IXL RESET Lock", MTX_DEF);
 387                         }
 388                         return (BUS_PROBE_DEFAULT);
 389                 }
 390                 ent++;
 391         }
 392         return (ENXIO);
 393 }
 394
 395 /*********************************************************************
 396  *  Device initialization routine
 397  *
 398  *  The attach entry point is called when the driver is being loaded.
 399  *  This routine identifies the type of hardware, allocates all resources
 400  *  and initializes the hardware.
 401  *
 402  *  return 0 on success, positive on failure
 403  *********************************************************************/
 404
 405 static int
 406 ixl_attach(device_t dev)
 407 {
 408         struct ixl_pf   *pf;
 409         struct i40e_hw  *hw;
 410         struct ixl_vsi *vsi;
 411         u16             bus;
 412         int             error = 0;
 413 #ifdef PCI_IOV
 414         nvlist_t        *pf_schema, *vf_schema;
 415         int             iov_error;
 416 #endif
 417
 418         INIT_DEBUGOUT("ixl_attach: begin");
 419
 420         /* Allocate, clear, and link in our primary soft structure */
 421         pf = device_get_softc(dev);
 422         pf->dev = pf->osdep.dev = dev;
 423         hw = &pf->hw;
 424
 425         /*
 426         ** Note this assumes we have a single embedded VSI,
 427         ** this could be enhanced later to allocate multiple
 428         */
 429         vsi = &pf->vsi;
 430         vsi->dev = pf->dev;
 431
 432         /* Core Lock Init*/
 433         IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
 434
 435         /* Set up the timer callout */
 436         callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
 437
 438         /* Set up sysctls */
 439         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 440             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 441             OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 442             pf, 0, ixl_set_flowcntl, "I", "Flow Control");
 443
 444         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 445             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 446             OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 447             pf, 0, ixl_set_advertise, "I", "Advertised Speed");
 448
 449         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 450             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 451             OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
 452             pf, 0, ixl_current_speed, "A", "Current Port Speed");
 453
 454         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 455             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 456             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
 457             pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
 458
 459         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 460             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 461             OID_AUTO, "rx_itr", CTLFLAG_RW,
 462             &ixl_rx_itr, IXL_ITR_8K, "RX ITR");
 463
 464         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 465             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 466             OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
 467             &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
 468
 469         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 470             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 471             OID_AUTO, "tx_itr", CTLFLAG_RW,
 472             &ixl_tx_itr, IXL_ITR_4K, "TX ITR");
 473
 474         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 475             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 476             OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
 477             &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
 478
 479 #ifdef IXL_DEBUG_SYSCTL
 480         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 481             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 482             OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
 483             pf, 0, ixl_sysctl_link_status, "A", "Current Link Status");
 484
 485         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 486             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 487             OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
 488             pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
 489
 490         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 491             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 492             OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
 493             pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
 494
 495         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 496             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 497             OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
 498             pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
 499
 500         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 501             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 502             OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
 503             pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
 504 #endif
 505
 506         /* Save off the PCI information */
 507         hw->vendor_id = pci_get_vendor(dev);
 508         hw->device_id = pci_get_device(dev);
 509         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
 510         hw->subsystem_vendor_id =
 511             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 512         hw->subsystem_device_id =
 513             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 514
 515         hw->bus.device = pci_get_slot(dev);
 516         hw->bus.func = pci_get_function(dev);
 517
 518         pf->vc_debug_lvl = 1;
 519
 520         /* Do PCI setup - map BAR0, etc */
 521         if (ixl_allocate_pci_resources(pf)) {
 522                 device_printf(dev, "Allocation of PCI resources failed\n");
 523                 error = ENXIO;
 524                 goto err_out;
 525         }
 526
 527         /* Create for initial debugging use */
 528         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 529             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 530             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
 531             ixl_debug_info, "I", "Debug Information");
 532
 533
 534         /* Establish a clean starting point */
 535         i40e_clear_hw(hw);
 536         error = i40e_pf_reset(hw);
 537         if (error) {
 538                 device_printf(dev,"PF reset failure %x\n", error);
 539                 error = EIO;
 540                 goto err_out;
 541         }
 542
 543         /* Set admin queue parameters */
 544         hw->aq.num_arq_entries = IXL_AQ_LEN;
 545         hw->aq.num_asq_entries = IXL_AQ_LEN;
 546         hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
 547         hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
 548
 549         /* Initialize the shared code */
 550         error = i40e_init_shared_code(hw);
 551         if (error) {
 552                 device_printf(dev,"Unable to initialize the shared code\n");
 553                 error = EIO;
 554                 goto err_out;
 555         }
 556
 557         /* Set up the admin queue */
 558         error = i40e_init_adminq(hw);
 559         if (error) {
 560                 device_printf(dev, "The driver for the device stopped "
 561                     "because the NVM image is newer than expected.\n"
 562                     "You must install the most recent version of "
 563                     " the network driver.\n");
 564                 goto err_out;
 565         }
 566         device_printf(dev, "%s\n", ixl_fw_version_str(hw));
 567
 568         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
 569             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
 570                 device_printf(dev, "The driver for the device detected "
 571                     "a newer version of the NVM image than expected.\n"
 572                     "Please install the most recent version of the network driver.\n");
 573         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
 574             hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
 575                 device_printf(dev, "The driver for the device detected "
 576                     "an older version of the NVM image than expected.\n"
 577                     "Please update the NVM image.\n");
 578
 579         /* Clear PXE mode */
 580         i40e_clear_pxe_mode(hw);
 581
 582         /* Get capabilities from the device */
 583         error = ixl_get_hw_capabilities(pf);
 584         if (error) {
 585                 device_printf(dev, "HW capabilities failure!\n");
 586                 goto err_get_cap;
 587         }
 588
 589         /* Set up host memory cache */
 590         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 591             hw->func_caps.num_rx_qp, 0, 0);
 592         if (error) {
 593                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
 594                 goto err_get_cap;
 595         }
 596
 597         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 598         if (error) {
 599                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
 600                 goto err_mac_hmc;
 601         }
 602
 603         /* Disable LLDP from the firmware */
 604         i40e_aq_stop_lldp(hw, TRUE, NULL);
 605
 606         i40e_get_mac_addr(hw, hw->mac.addr);
 607         error = i40e_validate_mac_addr(hw->mac.addr);
 608         if (error) {
 609                 device_printf(dev, "validate_mac_addr failed: %d\n", error);
 610                 goto err_mac_hmc;
 611         }
 612         bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
 613         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 614
 615         /* Set up VSI and queues */
 616         if (ixl_setup_stations(pf) != 0) {
 617                 device_printf(dev, "setup stations failed!\n");
 618                 error = ENOMEM;
 619                 goto err_mac_hmc;
 620         }
 621
 622         /* Initialize mac filter list for VSI */
 623         SLIST_INIT(&vsi->ftl);
 624
 625         /* Set up interrupt routing here */
 626         if (pf->msix > 1)
 627                 error = ixl_assign_vsi_msix(pf);
 628         else
 629                 error = ixl_assign_vsi_legacy(pf);
 630         if (error)
 631                 goto err_late;
 632
 633         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 634             (hw->aq.fw_maj_ver < 4)) {
 635                 i40e_msec_delay(75);
 636                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 637                 if (error)
 638                         device_printf(dev, "link restart failed, aq_err=%d\n",
 639                             pf->hw.aq.asq_last_status);
 640         }
 641
 642         /* Determine link state */
 643         i40e_aq_get_link_info(hw, TRUE, NULL, NULL);
 644         pf->link_up = i40e_get_link_status(hw);
 645
 646         /* Setup OS specific network interface */
 647         if (ixl_setup_interface(dev, vsi) != 0) {
 648                 device_printf(dev, "interface setup failed!\n");
 649                 error = EIO;
 650                 goto err_late;
 651         }
 652
 653         error = ixl_switch_config(pf);
 654         if (error) {
 655                 device_printf(dev, "Initial switch config failed: %d\n", error);
 656                 goto err_mac_hmc;
 657         }
 658
 659         /* Limit phy interrupts to link and modules failure */
 660         error = i40e_aq_set_phy_int_mask(hw,
 661             I40E_AQ_EVENT_LINK_UPDOWN | I40E_AQ_EVENT_MODULE_QUAL_FAIL, NULL);
 662         if (error)
 663                 device_printf(dev, "set phy mask failed: %d\n", error);
 664
 665         /* Get the bus configuration and set the shared code */
 666         bus = ixl_get_bus_info(hw, dev);
 667         i40e_set_pci_config_data(hw, bus);
 668
 669         /* Initialize statistics */
 670         ixl_pf_reset_stats(pf);
 671         ixl_update_stats_counters(pf);
 672         ixl_add_hw_stats(pf);
 673
 674         /* Register for VLAN events */
 675         vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 676             ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 677         vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 678             ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 679
 680 #ifdef PCI_IOV
 681         /* SR-IOV is only supported when MSI-X is in use. */
 682         if (pf->msix > 1) {
 683                 pf_schema = pci_iov_schema_alloc_node();
 684                 vf_schema = pci_iov_schema_alloc_node();
 685                 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
 686                 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
 687                     IOV_SCHEMA_HASDEFAULT, TRUE);
 688                 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
 689                     IOV_SCHEMA_HASDEFAULT, FALSE);
 690                 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
 691                     IOV_SCHEMA_HASDEFAULT, FALSE);
 692
 693                 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
 694                 if (iov_error != 0)
 695                         device_printf(dev,
 696                             "Failed to initialize SR-IOV (error=%d)\n",
 697                             iov_error);
 698         }
 699 #endif
 700
 701 #ifdef DEV_NETMAP
 702         ixl_netmap_attach(vsi);
 703 #endif /* DEV_NETMAP */
 704
 705         INIT_DEBUGOUT("ixl_attach: end");
 706         return (0);
 707
 708 err_late:
 709         if (vsi->ifp != NULL)
 710                 if_free(vsi->ifp);
 711 err_mac_hmc:
 712         i40e_shutdown_lan_hmc(hw);
 713 err_get_cap:
 714         i40e_shutdown_adminq(hw);
 715 err_out:
 716         ixl_free_pci_resources(pf);
 717         ixl_free_vsi(vsi);
 718         IXL_PF_LOCK_DESTROY(pf);
 719         return (error);
 720 }
 721
 722 /*********************************************************************
 723  *  Device removal routine
 724  *
 725  *  The detach entry point is called when the driver is being removed.
 726  *  This routine stops the adapter and deallocates all the resources
 727  *  that were allocated for driver operation.
 728  *
 729  *  return 0 on success, positive on failure
 730  *********************************************************************/
 731
 732 static int
 733 ixl_detach(device_t dev)
 734 {
 735         struct ixl_pf           *pf = device_get_softc(dev);
 736         struct i40e_hw          *hw = &pf->hw;
 737         struct ixl_vsi          *vsi = &pf->vsi;
 738         struct ixl_queue        *que = vsi->queues;
 739         i40e_status             status;
 740 #ifdef PCI_IOV
 741         int                     error;
 742 #endif
 743
 744         INIT_DEBUGOUT("ixl_detach: begin");
 745
 746         /* Make sure VLANS are not using driver */
 747         if (vsi->ifp->if_vlantrunk != NULL) {
 748                 device_printf(dev,"Vlan in use, detach first\n");
 749                 return (EBUSY);
 750         }
 751
 752 #ifdef PCI_IOV
 753         error = pci_iov_detach(dev);
 754         if (error != 0) {
 755                 device_printf(dev, "SR-IOV in use; detach first.\n");
 756                 return (error);
 757         }
 758 #endif
 759
 760         ether_ifdetach(vsi->ifp);
 761         if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) {
 762                 IXL_PF_LOCK(pf);
 763                 ixl_stop(pf);
 764                 IXL_PF_UNLOCK(pf);
 765         }
 766
 767         for (int i = 0; i < vsi->num_queues; i++, que++) {
 768                 if (que->tq) {
 769                         taskqueue_drain(que->tq, &que->task);
 770                         taskqueue_drain(que->tq, &que->tx_task);
 771                         taskqueue_free(que->tq);
 772                 }
 773         }
 774
 775         /* Shutdown LAN HMC */
 776         status = i40e_shutdown_lan_hmc(hw);
 777         if (status)
 778                 device_printf(dev,
 779                     "Shutdown LAN HMC failed with code %d\n", status);
 780
 781         /* Shutdown admin queue */
 782         status = i40e_shutdown_adminq(hw);
 783         if (status)
 784                 device_printf(dev,
 785                     "Shutdown Admin queue failed with code %d\n", status);
 786
 787         /* Unregister VLAN events */
 788         if (vsi->vlan_attach != NULL)
 789                 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
 790         if (vsi->vlan_detach != NULL)
 791                 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
 792
 793         callout_drain(&pf->timer);
 794 #ifdef DEV_NETMAP
 795         netmap_detach(vsi->ifp);
 796 #endif /* DEV_NETMAP */
 797         ixl_free_pci_resources(pf);
 798         bus_generic_detach(dev);
 799         if_free(vsi->ifp);
 800         ixl_free_vsi(vsi);
 801         IXL_PF_LOCK_DESTROY(pf);
 802         return (0);
 803 }
 804
 805 /*********************************************************************
 806  *
 807  *  Shutdown entry point
 808  *
 809  **********************************************************************/
 810
 811 static int
 812 ixl_shutdown(device_t dev)
 813 {
 814         struct ixl_pf *pf = device_get_softc(dev);
 815         IXL_PF_LOCK(pf);
 816         ixl_stop(pf);
 817         IXL_PF_UNLOCK(pf);
 818         return (0);
 819 }
 820
 821
 822 /*********************************************************************
 823  *
 824  *  Get the hardware capabilities
 825  *
 826  **********************************************************************/
 827
 828 static int
 829 ixl_get_hw_capabilities(struct ixl_pf *pf)
 830 {
 831         struct i40e_aqc_list_capabilities_element_resp *buf;
 832         struct i40e_hw  *hw = &pf->hw;
 833         device_t        dev = pf->dev;
 834         int             error, len;
 835         u16             needed;
 836         bool            again = TRUE;
 837
 838         len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
 839 retry:
 840         if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
 841             malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 842                 device_printf(dev, "Unable to allocate cap memory\n");
 843                 return (ENOMEM);
 844         }
 845
 846         /* This populates the hw struct */
 847         error = i40e_aq_discover_capabilities(hw, buf, len,
 848             &needed, i40e_aqc_opc_list_func_capabilities, NULL);
 849         free(buf, M_DEVBUF);
 850         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
 851             (again == TRUE)) {
 852                 /* retry once with a larger buffer */
 853                 again = FALSE;
 854                 len = needed;
 855                 goto retry;
 856         } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
 857                 device_printf(dev, "capability discovery failed: %d\n",
 858                     pf->hw.aq.asq_last_status);
 859                 return (ENODEV);
 860         }
 861
 862         /* Capture this PF's starting queue pair */
 863         pf->qbase = hw->func_caps.base_queue;
 864
 865 #ifdef IXL_DEBUG
 866         device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, "
 867             "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
 868             hw->pf_id, hw->func_caps.num_vfs,
 869             hw->func_caps.num_msix_vectors,
 870             hw->func_caps.num_msix_vectors_vf,
 871             hw->func_caps.fd_filters_guaranteed,
 872             hw->func_caps.fd_filters_best_effort,
 873             hw->func_caps.num_tx_qp,
 874             hw->func_caps.num_rx_qp,
 875             hw->func_caps.base_queue);
 876 #endif
 877         return (error);
 878 }
 879
 880 static void
 881 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
 882 {
 883         device_t        dev = vsi->dev;
 884
 885         /* Enable/disable TXCSUM/TSO4 */
 886         if (!(ifp->if_capenable & IFCAP_TXCSUM)
 887             && !(ifp->if_capenable & IFCAP_TSO4)) {
 888                 if (mask & IFCAP_TXCSUM) {
 889                         ifp->if_capenable |= IFCAP_TXCSUM;
 890                         /* enable TXCSUM, restore TSO if previously enabled */
 891                         if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
 892                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 893                                 ifp->if_capenable |= IFCAP_TSO4;
 894                         }
 895                 }
 896                 else if (mask & IFCAP_TSO4) {
 897                         ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
 898                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 899                         device_printf(dev,
 900                             "TSO4 requires txcsum, enabling both...\n");
 901                 }
 902         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 903             && !(ifp->if_capenable & IFCAP_TSO4)) {
 904                 if (mask & IFCAP_TXCSUM)
 905                         ifp->if_capenable &= ~IFCAP_TXCSUM;
 906                 else if (mask & IFCAP_TSO4)
 907                         ifp->if_capenable |= IFCAP_TSO4;
 908         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 909             && (ifp->if_capenable & IFCAP_TSO4)) {
 910                 if (mask & IFCAP_TXCSUM) {
 911                         vsi->flags |= IXL_FLAGS_KEEP_TSO4;
 912                         ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
 913                         device_printf(dev,
 914                             "TSO4 requires txcsum, disabling both...\n");
 915                 } else if (mask & IFCAP_TSO4)
 916                         ifp->if_capenable &= ~IFCAP_TSO4;
 917         }
 918
 919         /* Enable/disable TXCSUM_IPV6/TSO6 */
 920         if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 921             && !(ifp->if_capenable & IFCAP_TSO6)) {
 922                 if (mask & IFCAP_TXCSUM_IPV6) {
 923                         ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
 924                         if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
 925                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 926                                 ifp->if_capenable |= IFCAP_TSO6;
 927                         }
 928                 } else if (mask & IFCAP_TSO6) {
 929                         ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 930                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 931                         device_printf(dev,
 932                             "TSO6 requires txcsum6, enabling both...\n");
 933                 }
 934         } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 935             && !(ifp->if_capenable & IFCAP_TSO6)) {
 936                 if (mask & IFCAP_TXCSUM_IPV6)
 937                         ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
 938                 else if (mask & IFCAP_TSO6)
 939                         ifp->if_capenable |= IFCAP_TSO6;
 940         } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 941             && (ifp->if_capenable & IFCAP_TSO6)) {
 942                 if (mask & IFCAP_TXCSUM_IPV6) {
 943                         vsi->flags |= IXL_FLAGS_KEEP_TSO6;
 944                         ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 945                         device_printf(dev,
 946                             "TSO6 requires txcsum6, disabling both...\n");
 947                 } else if (mask & IFCAP_TSO6)
 948                         ifp->if_capenable &= ~IFCAP_TSO6;
 949         }
 950 }
 951
 952 /*********************************************************************
 953  *  Ioctl entry point
 954  *
 955  *  ixl_ioctl is called when the user wants to configure the
 956  *  interface.
 957  *
 958  *  return 0 on success, positive on failure
 959  **********************************************************************/
 960
 961 static int
 962 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 963 {
 964         struct ixl_vsi  *vsi = ifp->if_softc;
 965         struct ixl_pf   *pf = vsi->back;
 966         struct ifreq    *ifr = (struct ifreq *) data;
 967 #if defined(INET) || defined(INET6)
 968         struct ifaddr *ifa = (struct ifaddr *)data;
 969         bool            avoid_reset = FALSE;
 970 #endif
 971         int             error = 0;
 972
 973         switch (command) {
 974
 975         case SIOCSIFADDR:
 976 #ifdef INET
 977                 if (ifa->ifa_addr->sa_family == AF_INET)
 978                         avoid_reset = TRUE;
 979 #endif
 980 #ifdef INET6
 981                 if (ifa->ifa_addr->sa_family == AF_INET6)
 982                         avoid_reset = TRUE;
 983 #endif
 984 #if defined(INET) || defined(INET6)
 985                 /*
 986                 ** Calling init results in link renegotiation,
 987                 ** so we avoid doing it when possible.
 988                 */
 989                 if (avoid_reset) {
 990                         ifp->if_flags |= IFF_UP;
 991                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 992                                 ixl_init(pf);
 993 #ifdef INET
 994                         if (!(ifp->if_flags & IFF_NOARP))
 995                                 arp_ifinit(ifp, ifa);
 996 #endif
 997                 } else
 998                         error = ether_ioctl(ifp, command, data);
 999                 break;
1000 #endif
1001         case SIOCSIFMTU:
1002                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1003                 if (ifr->ifr_mtu > IXL_MAX_FRAME -
1004                    ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
1005                         error = EINVAL;
1006                 } else {
1007                         IXL_PF_LOCK(pf);
1008                         ifp->if_mtu = ifr->ifr_mtu;
1009                         vsi->max_frame_size =
1010                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
1011                             + ETHER_VLAN_ENCAP_LEN;
1012                         ixl_init_locked(pf);
1013                         IXL_PF_UNLOCK(pf);
1014                 }
1015                 break;
1016         case SIOCSIFFLAGS:
1017                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1018                 IXL_PF_LOCK(pf);
1019                 if (ifp->if_flags & IFF_UP) {
1020                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1021                                 if ((ifp->if_flags ^ pf->if_flags) &
1022                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1023                                         ixl_set_promisc(vsi);
1024                                 }
1025                         } else
1026                                 ixl_init_locked(pf);
1027                 } else
1028                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1029                                 ixl_stop(pf);
1030                 pf->if_flags = ifp->if_flags;
1031                 IXL_PF_UNLOCK(pf);
1032                 break;
1033         case SIOCADDMULTI:
1034                 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1035                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1036                         IXL_PF_LOCK(pf);
1037                         ixl_disable_intr(vsi);
1038                         ixl_add_multi(vsi);
1039                         ixl_enable_intr(vsi);
1040                         IXL_PF_UNLOCK(pf);
1041                 }
1042                 break;
1043         case SIOCDELMULTI:
1044                 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1045                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1046                         IXL_PF_LOCK(pf);
1047                         ixl_disable_intr(vsi);
1048                         ixl_del_multi(vsi);
1049                         ixl_enable_intr(vsi);
1050                         IXL_PF_UNLOCK(pf);
1051                 }
1052                 break;
1053         case SIOCSIFMEDIA:
1054         case SIOCGIFMEDIA:
1055                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1056                 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1057                 break;
1058         case SIOCSIFCAP:
1059         {
1060                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1061                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1062
1063                 ixl_cap_txcsum_tso(vsi, ifp, mask);
1064
1065                 if (mask & IFCAP_RXCSUM)
1066                         ifp->if_capenable ^= IFCAP_RXCSUM;
1067                 if (mask & IFCAP_RXCSUM_IPV6)
1068                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1069                 if (mask & IFCAP_LRO)
1070                         ifp->if_capenable ^= IFCAP_LRO;
1071                 if (mask & IFCAP_VLAN_HWTAGGING)
1072                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1073                 if (mask & IFCAP_VLAN_HWFILTER)
1074                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1075                 if (mask & IFCAP_VLAN_HWTSO)
1076                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1077                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1078                         IXL_PF_LOCK(pf);
1079                         ixl_init_locked(pf);
1080                         IXL_PF_UNLOCK(pf);
1081                 }
1082                 VLAN_CAPABILITIES(ifp);
1083
1084                 break;
1085         }
1086
1087         default:
1088                 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1089                 error = ether_ioctl(ifp, command, data);
1090                 break;
1091         }
1092
1093         return (error);
1094 }
1095
1096
1097 /*********************************************************************
1098  *  Init entry point
1099  *
1100  *  This routine is used in two ways. It is used by the stack as
1101  *  init entry point in network interface structure. It is also used
1102  *  by the driver as a hw/sw initialization routine to get to a
1103  *  consistent state.
1104  *
1105  *  return 0 on success, positive on failure
1106  **********************************************************************/
1107
1108 static void
1109 ixl_init_locked(struct ixl_pf *pf)
1110 {
1111         struct i40e_hw  *hw = &pf->hw;
1112         struct ixl_vsi  *vsi = &pf->vsi;
1113         struct ifnet    *ifp = vsi->ifp;
1114         device_t        dev = pf->dev;
1115         struct i40e_filter_control_settings     filter;
1116         u8              tmpaddr[ETHER_ADDR_LEN];
1117         int             ret;
1118
1119         mtx_assert(&pf->pf_mtx, MA_OWNED);
1120         INIT_DEBUGOUT("ixl_init: begin");
1121         ixl_stop(pf);
1122
1123         /* Get the latest mac address... User might use a LAA */
1124         bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1125               I40E_ETH_LENGTH_OF_ADDRESS);
1126         if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1127             i40e_validate_mac_addr(tmpaddr)) {
1128                 bcopy(tmpaddr, hw->mac.addr,
1129                     I40E_ETH_LENGTH_OF_ADDRESS);
1130                 ret = i40e_aq_mac_address_write(hw,
1131                     I40E_AQC_WRITE_TYPE_LAA_ONLY,
1132                     hw->mac.addr, NULL);
1133                 if (ret) {
1134                         device_printf(dev, "LLA address"
1135                          "change failed!!\n");
1136                         return;
1137                 }
1138         }
1139
1140         /* Set the various hardware offload abilities */
1141         ifp->if_hwassist = 0;
1142         if (ifp->if_capenable & IFCAP_TSO)
1143                 ifp->if_hwassist |= CSUM_TSO;
1144         if (ifp->if_capenable & IFCAP_TXCSUM)
1145                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1146         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1147                 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1148
1149         /* Set up the device filtering */
1150         bzero(&filter, sizeof(filter));
1151         filter.enable_ethtype = TRUE;
1152         filter.enable_macvlan = TRUE;
1153 #ifdef IXL_FDIR
1154         filter.enable_fdir = TRUE;
1155 #endif
1156         if (i40e_set_filter_control(hw, &filter))
1157                 device_printf(dev, "set_filter_control() failed\n");
1158
1159         /* Set up RSS */
1160         ixl_config_rss(vsi);
1161
1162         /*
1163         ** Prepare the VSI: rings, hmc contexts, etc...
1164         */
1165         if (ixl_initialize_vsi(vsi)) {
1166                 device_printf(dev, "initialize vsi failed!!\n");
1167                 return;
1168         }
1169
1170         /* Add protocol filters to list */
1171         ixl_init_filters(vsi);
1172
1173         /* Setup vlan's if needed */
1174         ixl_setup_vlan_filters(vsi);
1175
1176         /* Start the local timer */
1177         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1178
1179         /* Set up MSI/X routing and the ITR settings */
1180         if (ixl_enable_msix) {
1181                 ixl_configure_msix(pf);
1182                 ixl_configure_itr(pf);
1183         } else
1184                 ixl_configure_legacy(pf);
1185
1186         ixl_enable_rings(vsi);
1187
1188         i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1189
1190         ixl_reconfigure_filters(vsi);
1191
1192         /* Set MTU in hardware*/
1193         int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size,
1194             TRUE, 0, NULL);
1195         if (aq_error)
1196                 device_printf(vsi->dev,
1197                         "aq_set_mac_config in init error, code %d\n",
1198                     aq_error);
1199
1200         /* And now turn on interrupts */
1201         ixl_enable_intr(vsi);
1202
1203         /* Now inform the stack we're ready */
1204         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1205         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1206
1207         return;
1208 }
1209
1210 static void
1211 ixl_init(void *arg)
1212 {
1213         struct ixl_pf *pf = arg;
1214
1215         IXL_PF_LOCK(pf);
1216         ixl_init_locked(pf);
1217         IXL_PF_UNLOCK(pf);
1218         return;
1219 }
1220
1221 /*
1222 **
1223 ** MSIX Interrupt Handlers and Tasklets
1224 **
1225 */
1226 static void
1227 ixl_handle_que(void *context, int pending)
1228 {
1229         struct ixl_queue *que = context;
1230         struct ixl_vsi *vsi = que->vsi;
1231         struct i40e_hw  *hw = vsi->hw;
1232         struct tx_ring  *txr = &que->txr;
1233         struct ifnet    *ifp = vsi->ifp;
1234         bool            more;
1235
1236         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1237                 more = ixl_rxeof(que, IXL_RX_LIMIT);
1238                 IXL_TX_LOCK(txr);
1239                 ixl_txeof(que);
1240                 if (!drbr_empty(ifp, txr->br))
1241                         ixl_mq_start_locked(ifp, txr);
1242                 IXL_TX_UNLOCK(txr);
1243                 if (more) {
1244                         taskqueue_enqueue(que->tq, &que->task);
1245                         return;
1246                 }
1247         }
1248
1249         /* Reenable this interrupt - hmmm */
1250         ixl_enable_queue(hw, que->me);
1251         return;
1252 }
1253
1254
1255 /*********************************************************************
1256  *
1257  *  Legacy Interrupt Service routine
1258  *
1259  **********************************************************************/
1260 void
1261 ixl_intr(void *arg)
1262 {
1263         struct ixl_pf           *pf = arg;
1264         struct i40e_hw          *hw =  &pf->hw;
1265         struct ixl_vsi          *vsi = &pf->vsi;
1266         struct ixl_queue        *que = vsi->queues;
1267         struct ifnet            *ifp = vsi->ifp;
1268         struct tx_ring          *txr = &que->txr;
1269         u32                     reg, icr0, mask;
1270         bool                    more_tx, more_rx;
1271
1272         ++que->irqs;
1273
1274         /* Protect against spurious interrupts */
1275         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1276                 return;
1277
1278         icr0 = rd32(hw, I40E_PFINT_ICR0);
1279
1280         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1281         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1282         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1283
1284         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1285
1286 #ifdef PCI_IOV
1287         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1288                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1289 #endif
1290
1291         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1292                 taskqueue_enqueue(pf->tq, &pf->adminq);
1293                 return;
1294         }
1295
1296         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1297
1298         IXL_TX_LOCK(txr);
1299         more_tx = ixl_txeof(que);
1300         if (!drbr_empty(vsi->ifp, txr->br))
1301                 more_tx = 1;
1302         IXL_TX_UNLOCK(txr);
1303
1304         /* re-enable other interrupt causes */
1305         wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1306
1307         /* And now the queues */
1308         reg = rd32(hw, I40E_QINT_RQCTL(0));
1309         reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1310         wr32(hw, I40E_QINT_RQCTL(0), reg);
1311
1312         reg = rd32(hw, I40E_QINT_TQCTL(0));
1313         reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1314         reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1315         wr32(hw, I40E_QINT_TQCTL(0), reg);
1316
1317         ixl_enable_legacy(hw);
1318
1319         return;
1320 }
1321
1322
1323 /*********************************************************************
1324  *
1325  *  MSIX VSI Interrupt Service routine
1326  *
1327  **********************************************************************/
1328 void
1329 ixl_msix_que(void *arg)
1330 {
1331         struct ixl_queue        *que = arg;
1332         struct ixl_vsi  *vsi = que->vsi;
1333         struct i40e_hw  *hw = vsi->hw;
1334         struct tx_ring  *txr = &que->txr;
1335         bool            more_tx, more_rx;
1336
1337         /* Protect against spurious interrupts */
1338         if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1339                 return;
1340
1341         ++que->irqs;
1342
1343         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1344
1345         IXL_TX_LOCK(txr);
1346         more_tx = ixl_txeof(que);
1347         /*
1348         ** Make certain that if the stack
1349         ** has anything queued the task gets
1350         ** scheduled to handle it.
1351         */
1352         if (!drbr_empty(vsi->ifp, txr->br))
1353                 more_tx = 1;
1354         IXL_TX_UNLOCK(txr);
1355
1356         ixl_set_queue_rx_itr(que);
1357         ixl_set_queue_tx_itr(que);
1358
1359         if (more_tx || more_rx)
1360                 taskqueue_enqueue(que->tq, &que->task);
1361         else
1362                 ixl_enable_queue(hw, que->me);
1363
1364         return;
1365 }
1366
1367
1368 /*********************************************************************
1369  *
1370  *  MSIX Admin Queue Interrupt Service routine
1371  *
1372  **********************************************************************/
1373 static void
1374 ixl_msix_adminq(void *arg)
1375 {
1376         struct ixl_pf   *pf = arg;
1377         struct i40e_hw  *hw = &pf->hw;
1378         u32             reg, mask;
1379
1380         ++pf->admin_irq;
1381
1382         reg = rd32(hw, I40E_PFINT_ICR0);
1383         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1384
1385         /* Check on the cause */
1386         if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
1387                 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
1388
1389         if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1390                 ixl_handle_mdd_event(pf);
1391                 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
1392         }
1393
1394 #ifdef PCI_IOV
1395         if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1396                 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1397                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1398         }
1399 #endif
1400
1401         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1402         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1403         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1404
1405         taskqueue_enqueue(pf->tq, &pf->adminq);
1406         return;
1407 }
1408
1409 /*********************************************************************
1410  *
1411  *  Media Ioctl callback
1412  *
1413  *  This routine is called whenever the user queries the status of
1414  *  the interface using ifconfig.
1415  *
1416  **********************************************************************/
1417 static void
1418 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1419 {
1420         struct ixl_vsi  *vsi = ifp->if_softc;
1421         struct ixl_pf   *pf = vsi->back;
1422         struct i40e_hw  *hw = &pf->hw;
1423
1424         INIT_DEBUGOUT("ixl_media_status: begin");
1425         IXL_PF_LOCK(pf);
1426
1427         hw->phy.get_link_info = TRUE;
1428         pf->link_up = i40e_get_link_status(hw);
1429         ixl_update_link_status(pf);
1430
1431         ifmr->ifm_status = IFM_AVALID;
1432         ifmr->ifm_active = IFM_ETHER;
1433
1434         if (!pf->link_up) {
1435                 IXL_PF_UNLOCK(pf);
1436                 return;
1437         }
1438
1439         ifmr->ifm_status |= IFM_ACTIVE;
1440         /* Hardware is always full-duplex */
1441         ifmr->ifm_active |= IFM_FDX;
1442
1443         switch (hw->phy.link_info.phy_type) {
1444                 /* 100 M */
1445                 case I40E_PHY_TYPE_100BASE_TX:
1446                         ifmr->ifm_active |= IFM_100_TX;
1447                         break;
1448                 /* 1 G */
1449                 case I40E_PHY_TYPE_1000BASE_T:
1450                         ifmr->ifm_active |= IFM_1000_T;
1451                         break;
1452                 case I40E_PHY_TYPE_1000BASE_SX:
1453                         ifmr->ifm_active |= IFM_1000_SX;
1454                         break;
1455                 case I40E_PHY_TYPE_1000BASE_LX:
1456                         ifmr->ifm_active |= IFM_1000_LX;
1457                         break;
1458                 /* 10 G */
1459                 case I40E_PHY_TYPE_10GBASE_CR1:
1460                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1461                 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1462                 /* Using this until a real KR media type */
1463                 case I40E_PHY_TYPE_10GBASE_KR:
1464                 case I40E_PHY_TYPE_10GBASE_KX4:
1465                         ifmr->ifm_active |= IFM_10G_TWINAX;
1466                         break;
1467                 case I40E_PHY_TYPE_10GBASE_SR:
1468                         ifmr->ifm_active |= IFM_10G_SR;
1469                         break;
1470                 case I40E_PHY_TYPE_10GBASE_LR:
1471                         ifmr->ifm_active |= IFM_10G_LR;
1472                         break;
1473                 case I40E_PHY_TYPE_10GBASE_T:
1474                         ifmr->ifm_active |= IFM_10G_T;
1475                         break;
1476                 /* 40 G */
1477                 case I40E_PHY_TYPE_40GBASE_CR4:
1478                 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1479                         ifmr->ifm_active |= IFM_40G_CR4;
1480                         break;
1481                 case I40E_PHY_TYPE_40GBASE_SR4:
1482                         ifmr->ifm_active |= IFM_40G_SR4;
1483                         break;
1484                 case I40E_PHY_TYPE_40GBASE_LR4:
1485                         ifmr->ifm_active |= IFM_40G_LR4;
1486                         break;
1487                 /*
1488                 ** Set these to CR4 because OS does not
1489                 ** have types available yet.
1490                 */
1491                 case I40E_PHY_TYPE_40GBASE_KR4:
1492                 case I40E_PHY_TYPE_XLAUI:
1493                 case I40E_PHY_TYPE_XLPPI:
1494                 case I40E_PHY_TYPE_40GBASE_AOC:
1495                         ifmr->ifm_active |= IFM_40G_CR4;
1496                         break;
1497                 default:
1498                         ifmr->ifm_active |= IFM_UNKNOWN;
1499                         break;
1500         }
1501         /* Report flow control status as well */
1502         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1503                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1504         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1505                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1506
1507         IXL_PF_UNLOCK(pf);
1508
1509         return;
1510 }
1511
1512 /*********************************************************************
1513  *
1514  *  Media Ioctl callback
1515  *
1516  *  This routine is called when the user changes speed/duplex using
1517  *  media/mediopt option with ifconfig.
1518  *
1519  **********************************************************************/
1520 static int
1521 ixl_media_change(struct ifnet * ifp)
1522 {
1523         struct ixl_vsi *vsi = ifp->if_softc;
1524         struct ifmedia *ifm = &vsi->media;
1525
1526         INIT_DEBUGOUT("ixl_media_change: begin");
1527
1528         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1529                 return (EINVAL);
1530
1531         if_printf(ifp, "Media change is currently not supported.\n");
1532
1533         return (ENODEV);
1534 }
1535
1536
1537 #ifdef IXL_FDIR
1538 /*
1539 ** ATR: Application Targetted Receive - creates a filter
1540 **      based on TX flow info that will keep the receive
1541 **      portion of the flow on the same queue. Based on the
1542 **      implementation this is only available for TCP connections
1543 */
1544 void
1545 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1546 {
1547         struct ixl_vsi                  *vsi = que->vsi;
1548         struct tx_ring                  *txr = &que->txr;
1549         struct i40e_filter_program_desc *FDIR;
1550         u32                             ptype, dtype;
1551         int                             idx;
1552
1553         /* check if ATR is enabled and sample rate */
1554         if ((!ixl_enable_fdir) || (!txr->atr_rate))
1555                 return;
1556         /*
1557         ** We sample all TCP SYN/FIN packets,
1558         ** or at the selected sample rate
1559         */
1560         txr->atr_count++;
1561         if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1562             (txr->atr_count < txr->atr_rate))
1563                 return;
1564         txr->atr_count = 0;
1565
1566         /* Get a descriptor to use */
1567         idx = txr->next_avail;
1568         FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1569         if (++idx == que->num_desc)
1570                 idx = 0;
1571         txr->avail--;
1572         txr->next_avail = idx;
1573
1574         ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1575             I40E_TXD_FLTR_QW0_QINDEX_MASK;
1576
1577         ptype |= (etype == ETHERTYPE_IP) ?
1578             (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1579             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1580             (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1581             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1582
1583         ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1584
1585         dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1586
1587         /*
1588         ** We use the TCP TH_FIN as a trigger to remove
1589         ** the filter, otherwise its an update.
1590         */
1591         dtype |= (th->th_flags & TH_FIN) ?
1592             (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1593             I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1594             (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1595             I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1596
1597         dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1598             I40E_TXD_FLTR_QW1_DEST_SHIFT;
1599
1600         dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1601             I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1602
1603         FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1604         FDIR->dtype_cmd_cntindex = htole32(dtype);
1605         return;
1606 }
1607 #endif
1608
1609
1610 static void
1611 ixl_set_promisc(struct ixl_vsi *vsi)
1612 {
1613         struct ifnet    *ifp = vsi->ifp;
1614         struct i40e_hw  *hw = vsi->hw;
1615         int             err, mcnt = 0;
1616         bool            uni = FALSE, multi = FALSE;
1617
1618         if (ifp->if_flags & IFF_ALLMULTI)
1619                 multi = TRUE;
1620         else { /* Need to count the multicast addresses */
1621                 struct  ifmultiaddr *ifma;
1622                 if_maddr_rlock(ifp);
1623                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1624                         if (ifma->ifma_addr->sa_family != AF_LINK)
1625                                 continue;
1626                         if (mcnt == MAX_MULTICAST_ADDR)
1627                                 break;
1628                         mcnt++;
1629                 }
1630                 if_maddr_runlock(ifp);
1631         }
1632
1633         if (mcnt >= MAX_MULTICAST_ADDR)
1634                 multi = TRUE;
1635         if (ifp->if_flags & IFF_PROMISC)
1636                 uni = TRUE;
1637
1638         err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1639             vsi->seid, uni, NULL);
1640         err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1641             vsi->seid, multi, NULL);
1642         return;
1643 }
1644
1645 /*********************************************************************
1646  *      Filter Routines
1647  *
1648  *      Routines for multicast and vlan filter management.
1649  *
1650  *********************************************************************/
1651 static void
1652 ixl_add_multi(struct ixl_vsi *vsi)
1653 {
1654         struct  ifmultiaddr     *ifma;
1655         struct ifnet            *ifp = vsi->ifp;
1656         struct i40e_hw          *hw = vsi->hw;
1657         int                     mcnt = 0, flags;
1658
1659         IOCTL_DEBUGOUT("ixl_add_multi: begin");
1660
1661         if_maddr_rlock(ifp);
1662         /*
1663         ** First just get a count, to decide if we
1664         ** we simply use multicast promiscuous.
1665         */
1666         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1667                 if (ifma->ifma_addr->sa_family != AF_LINK)
1668                         continue;
1669                 mcnt++;
1670         }
1671         if_maddr_runlock(ifp);
1672
1673         if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1674                 /* delete existing MC filters */
1675                 ixl_del_hw_filters(vsi, mcnt);
1676                 i40e_aq_set_vsi_multicast_promiscuous(hw,
1677                     vsi->seid, TRUE, NULL);
1678                 return;
1679         }
1680
1681         mcnt = 0;
1682         if_maddr_rlock(ifp);
1683         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1684                 if (ifma->ifma_addr->sa_family != AF_LINK)
1685                         continue;
1686                 ixl_add_mc_filter(vsi,
1687                     (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1688                 mcnt++;
1689         }
1690         if_maddr_runlock(ifp);
1691         if (mcnt > 0) {
1692                 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1693                 ixl_add_hw_filters(vsi, flags, mcnt);
1694         }
1695
1696         IOCTL_DEBUGOUT("ixl_add_multi: end");
1697         return;
1698 }
1699
1700 static void
1701 ixl_del_multi(struct ixl_vsi *vsi)
1702 {
1703         struct ifnet            *ifp = vsi->ifp;
1704         struct ifmultiaddr      *ifma;
1705         struct ixl_mac_filter   *f;
1706         int                     mcnt = 0;
1707         bool            match = FALSE;
1708
1709         IOCTL_DEBUGOUT("ixl_del_multi: begin");
1710
1711         /* Search for removed multicast addresses */
1712         if_maddr_rlock(ifp);
1713         SLIST_FOREACH(f, &vsi->ftl, next) {
1714                 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1715                         match = FALSE;
1716                         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1717                                 if (ifma->ifma_addr->sa_family != AF_LINK)
1718                                         continue;
1719                                 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
1720                                 if (cmp_etheraddr(f->macaddr, mc_addr)) {
1721                                         match = TRUE;
1722                                         break;
1723                                 }
1724                         }
1725                         if (match == FALSE) {
1726                                 f->flags |= IXL_FILTER_DEL;
1727                                 mcnt++;
1728                         }
1729                 }
1730         }
1731         if_maddr_runlock(ifp);
1732
1733         if (mcnt > 0)
1734                 ixl_del_hw_filters(vsi, mcnt);
1735 }
1736
1737
1738 /*********************************************************************
1739  *  Timer routine
1740  *
1741  *  This routine checks for link status,updates statistics,
1742  *  and runs the watchdog check.
1743  *
1744  **********************************************************************/
1745
1746 static void
1747 ixl_local_timer(void *arg)
1748 {
1749         struct ixl_pf           *pf = arg;
1750         struct i40e_hw          *hw = &pf->hw;
1751         struct ixl_vsi          *vsi = &pf->vsi;
1752         struct ixl_queue        *que = vsi->queues;
1753         device_t                dev = pf->dev;
1754         int                     hung = 0;
1755         u32                     mask;
1756
1757         mtx_assert(&pf->pf_mtx, MA_OWNED);
1758
1759         /* Fire off the adminq task */
1760         taskqueue_enqueue(pf->tq, &pf->adminq);
1761
1762         /* Update stats */
1763         ixl_update_stats_counters(pf);
1764
1765         /*
1766         ** Check status of the queues
1767         */
1768         mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
1769                 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
1770
1771         for (int i = 0; i < vsi->num_queues; i++,que++) {
1772                 /* Any queues with outstanding work get a sw irq */
1773                 if (que->busy)
1774                         wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
1775                 /*
1776                 ** Each time txeof runs without cleaning, but there
1777                 ** are uncleaned descriptors it increments busy. If
1778                 ** we get to 5 we declare it hung.
1779                 */
1780                 if (que->busy == IXL_QUEUE_HUNG) {
1781                         ++hung;
1782                         /* Mark the queue as inactive */
1783                         vsi->active_queues &= ~((u64)1 << que->me);
1784                         continue;
1785                 } else {
1786                         /* Check if we've come back from hung */
1787                         if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
1788                                 vsi->active_queues |= ((u64)1 << que->me);
1789                 }
1790                 if (que->busy >= IXL_MAX_TX_BUSY) {
1791 #ifdef IXL_DEBUG
1792                         device_printf(dev,"Warning queue %d "
1793                             "appears to be hung!\n", i);
1794 #endif
1795                         que->busy = IXL_QUEUE_HUNG;
1796                         ++hung;
1797                 }
1798         }
1799         /* Only reinit if all queues show hung */
1800         if (hung == vsi->num_queues)
1801                 goto hung;
1802
1803         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1804         return;
1805
1806 hung:
1807         device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
1808         ixl_init_locked(pf);
1809 }
1810
1811 /*
1812 ** Note: this routine updates the OS on the link state
1813 **      the real check of the hardware only happens with
1814 **      a link interrupt.
1815 */
1816 static void
1817 ixl_update_link_status(struct ixl_pf *pf)
1818 {
1819         struct ixl_vsi          *vsi = &pf->vsi;
1820         struct i40e_hw          *hw = &pf->hw;
1821         struct ifnet            *ifp = vsi->ifp;
1822         device_t                dev = pf->dev;
1823
1824         if (pf->link_up){
1825                 if (vsi->link_active == FALSE) {
1826                         pf->fc = hw->fc.current_mode;
1827                         if (bootverbose) {
1828                                 device_printf(dev,"Link is up %d Gbps %s,"
1829                                     " Flow Control: %s\n",
1830                                     ((pf->link_speed ==
1831                                     I40E_LINK_SPEED_40GB)? 40:10),
1832                                     "Full Duplex", ixl_fc_string[pf->fc]);
1833                         }
1834                         vsi->link_active = TRUE;
1835                         /*
1836                         ** Warn user if link speed on NPAR enabled
1837                         ** partition is not at least 10GB
1838                         */
1839                         if (hw->func_caps.npar_enable &&
1840                            (hw->phy.link_info.link_speed ==
1841                            I40E_LINK_SPEED_1GB ||
1842                            hw->phy.link_info.link_speed ==
1843                            I40E_LINK_SPEED_100MB))
1844                                 device_printf(dev, "The partition detected"
1845                                     "link speed that is less than 10Gbps\n");
1846                         if_link_state_change(ifp, LINK_STATE_UP);
1847                 }
1848         } else { /* Link down */
1849                 if (vsi->link_active == TRUE) {
1850                         if (bootverbose)
1851                                 device_printf(dev,"Link is Down\n");
1852                         if_link_state_change(ifp, LINK_STATE_DOWN);
1853                         vsi->link_active = FALSE;
1854                 }
1855         }
1856
1857         return;
1858 }
1859
1860 /*********************************************************************
1861  *
1862  *  This routine disables all traffic on the adapter by issuing a
1863  *  global reset on the MAC and deallocates TX/RX buffers.
1864  *
1865  **********************************************************************/
1866
1867 static void
1868 ixl_stop(struct ixl_pf *pf)
1869 {
1870         struct ixl_vsi  *vsi = &pf->vsi;
1871         struct ifnet    *ifp = vsi->ifp;
1872
1873         mtx_assert(&pf->pf_mtx, MA_OWNED);
1874
1875         INIT_DEBUGOUT("ixl_stop: begin\n");
1876         if (pf->num_vfs == 0)
1877                 ixl_disable_intr(vsi);
1878         else
1879                 ixl_disable_rings_intr(vsi);
1880         ixl_disable_rings(vsi);
1881
1882         /* Tell the stack that the interface is no longer active */
1883         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1884
1885         /* Stop the local timer */
1886         callout_stop(&pf->timer);
1887
1888         return;
1889 }
1890
1891
1892 /*********************************************************************
1893  *
1894  *  Setup MSIX Interrupt resources and handlers for the VSI
1895  *
1896  **********************************************************************/
1897 static int
1898 ixl_assign_vsi_legacy(struct ixl_pf *pf)
1899 {
1900         device_t        dev = pf->dev;
1901         struct          ixl_vsi *vsi = &pf->vsi;
1902         struct          ixl_queue *que = vsi->queues;
1903         int             error, rid = 0;
1904
1905         if (pf->msix == 1)
1906                 rid = 1;
1907         pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1908             &rid, RF_SHAREABLE | RF_ACTIVE);
1909         if (pf->res == NULL) {
1910                 device_printf(dev,"Unable to allocate"
1911                     " bus resource: vsi legacy/msi interrupt\n");
1912                 return (ENXIO);
1913         }
1914
1915         /* Set the handler function */
1916         error = bus_setup_intr(dev, pf->res,
1917             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1918             ixl_intr, pf, &pf->tag);
1919         if (error) {
1920                 pf->res = NULL;
1921                 device_printf(dev, "Failed to register legacy/msi handler");
1922                 return (error);
1923         }
1924         bus_describe_intr(dev, pf->res, pf->tag, "irq0");
1925         TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
1926         TASK_INIT(&que->task, 0, ixl_handle_que, que);
1927         que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
1928             taskqueue_thread_enqueue, &que->tq);
1929         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1930             device_get_nameunit(dev));
1931         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1932
1933 #ifdef PCI_IOV
1934         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1935 #endif
1936
1937         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1938             taskqueue_thread_enqueue, &pf->tq);
1939         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1940             device_get_nameunit(dev));
1941
1942         return (0);
1943 }
1944
1945
1946 /*********************************************************************
1947  *
1948  *  Setup MSIX Interrupt resources and handlers for the VSI
1949  *
1950  **********************************************************************/
1951 static int
1952 ixl_assign_vsi_msix(struct ixl_pf *pf)
1953 {
1954         device_t        dev = pf->dev;
1955         struct          ixl_vsi *vsi = &pf->vsi;
1956         struct          ixl_queue *que = vsi->queues;
1957         struct          tx_ring  *txr;
1958         int             error, rid, vector = 0;
1959
1960         /* Admin Que is vector 0*/
1961         rid = vector + 1;
1962         pf->res = bus_alloc_resource_any(dev,
1963             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
1964         if (!pf->res) {
1965                 device_printf(dev,"Unable to allocate"
1966             " bus resource: Adminq interrupt [%d]\n", rid);
1967                 return (ENXIO);
1968         }
1969         /* Set the adminq vector and handler */
1970         error = bus_setup_intr(dev, pf->res,
1971             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1972             ixl_msix_adminq, pf, &pf->tag);
1973         if (error) {
1974                 pf->res = NULL;
1975                 device_printf(dev, "Failed to register Admin que handler");
1976                 return (error);
1977         }
1978         bus_describe_intr(dev, pf->res, pf->tag, "aq");
1979         pf->admvec = vector;
1980         /* Tasklet for Admin Queue */
1981         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1982
1983 #ifdef PCI_IOV
1984         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1985 #endif
1986
1987         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1988             taskqueue_thread_enqueue, &pf->tq);
1989         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1990             device_get_nameunit(pf->dev));
1991         ++vector;
1992
1993         /* Now set up the stations */
1994         for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
1995                 int cpu_id = i;
1996                 rid = vector + 1;
1997                 txr = &que->txr;
1998                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1999                     RF_SHAREABLE | RF_ACTIVE);
2000                 if (que->res == NULL) {
2001                         device_printf(dev,"Unable to allocate"
2002                             " bus resource: que interrupt [%d]\n", vector);
2003                         return (ENXIO);
2004                 }
2005                 /* Set the handler function */
2006                 error = bus_setup_intr(dev, que->res,
2007                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2008                     ixl_msix_que, que, &que->tag);
2009                 if (error) {
2010                         que->res = NULL;
2011                         device_printf(dev, "Failed to register que handler");
2012                         return (error);
2013                 }
2014                 bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2015                 /* Bind the vector to a CPU */
2016 #ifdef RSS
2017                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2018 #endif
2019                 bus_bind_intr(dev, que->res, cpu_id);
2020                 que->msix = vector;
2021                 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2022                 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2023                 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2024                     taskqueue_thread_enqueue, &que->tq);
2025 #ifdef RSS
2026                 taskqueue_start_threads_pinned(&que->tq, 1, PI_NET,
2027                     cpu_id, "%s (bucket %d)",
2028                     device_get_nameunit(dev), cpu_id);
2029 #else
2030                 taskqueue_start_threads(&que->tq, 1, PI_NET,
2031                     "%s que", device_get_nameunit(dev));
2032 #endif
2033         }
2034
2035         return (0);
2036 }
2037
2038
2039 /*
2040  * Allocate MSI/X vectors
2041  */
2042 static int
2043 ixl_init_msix(struct ixl_pf *pf)
2044 {
2045         device_t dev = pf->dev;
2046         int rid, want, vectors, queues, available;
2047
2048         /* Override by tuneable */
2049         if (ixl_enable_msix == 0)
2050                 goto msi;
2051
2052         /*
2053         ** When used in a virtualized environment
2054         ** PCI BUSMASTER capability may not be set
2055         ** so explicity set it here and rewrite
2056         ** the ENABLE in the MSIX control register
2057         ** at this point to cause the host to
2058         ** successfully initialize us.
2059         */
2060         {
2061                 u16 pci_cmd_word;
2062                 int msix_ctrl;
2063                 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2064                 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2065                 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2066                 pci_find_cap(dev, PCIY_MSIX, &rid);
2067                 rid += PCIR_MSIX_CTRL;
2068                 msix_ctrl = pci_read_config(dev, rid, 2);
2069                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2070                 pci_write_config(dev, rid, msix_ctrl, 2);
2071         }
2072
2073         /* First try MSI/X */
2074         rid = PCIR_BAR(IXL_BAR);
2075         pf->msix_mem = bus_alloc_resource_any(dev,
2076             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2077         if (!pf->msix_mem) {
2078                 /* May not be enabled */
2079                 device_printf(pf->dev,
2080                     "Unable to map MSIX table \n");
2081                 goto msi;
2082         }
2083
2084         available = pci_msix_count(dev);
2085         if (available == 0) { /* system has msix disabled */
2086                 bus_release_resource(dev, SYS_RES_MEMORY,
2087                     rid, pf->msix_mem);
2088                 pf->msix_mem = NULL;
2089                 goto msi;
2090         }
2091
2092         /* Figure out a reasonable auto config value */
2093         queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2094
2095         /* Override with hardcoded value if sane */
2096         if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2097                 queues = ixl_max_queues;
2098
2099 #ifdef  RSS
2100         /* If we're doing RSS, clamp at the number of RSS buckets */
2101         if (queues > rss_getnumbuckets())
2102                 queues = rss_getnumbuckets();
2103 #endif
2104
2105         /*
2106         ** Want one vector (RX/TX pair) per queue
2107         ** plus an additional for the admin queue.
2108         */
2109         want = queues + 1;
2110         if (want <= available)  /* Have enough */
2111                 vectors = want;
2112         else {
2113                 device_printf(pf->dev,
2114                     "MSIX Configuration Problem, "
2115                     "%d vectors available but %d wanted!\n",
2116                     available, want);
2117                 return (0); /* Will go to Legacy setup */
2118         }
2119
2120         if (pci_alloc_msix(dev, &vectors) == 0) {
2121                 device_printf(pf->dev,
2122                     "Using MSIX interrupts with %d vectors\n", vectors);
2123                 pf->msix = vectors;
2124                 pf->vsi.num_queues = queues;
2125 #ifdef RSS
2126                 /*
2127                  * If we're doing RSS, the number of queues needs to
2128                  * match the number of RSS buckets that are configured.
2129                  *
2130                  * + If there's more queues than RSS buckets, we'll end
2131                  *   up with queues that get no traffic.
2132                  *
2133                  * + If there's more RSS buckets than queues, we'll end
2134                  *   up having multiple RSS buckets map to the same queue,
2135                  *   so there'll be some contention.
2136                  */
2137                 if (queues != rss_getnumbuckets()) {
2138                         device_printf(dev,
2139                             "%s: queues (%d) != RSS buckets (%d)"
2140                             "; performance will be impacted.\n",
2141                             __func__, queues, rss_getnumbuckets());
2142                 }
2143 #endif
2144                 return (vectors);
2145         }
2146 msi:
2147         vectors = pci_msi_count(dev);
2148         pf->vsi.num_queues = 1;
2149         pf->msix = 1;
2150         ixl_max_queues = 1;
2151         ixl_enable_msix = 0;
2152         if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2153                 device_printf(pf->dev,"Using an MSI interrupt\n");
2154         else {
2155                 pf->msix = 0;
2156                 device_printf(pf->dev,"Using a Legacy interrupt\n");
2157         }
2158         return (vectors);
2159 }
2160
2161
2162 /*
2163  * Plumb MSI/X vectors
2164  */
2165 static void
2166 ixl_configure_msix(struct ixl_pf *pf)
2167 {
2168         struct i40e_hw  *hw = &pf->hw;
2169         struct ixl_vsi *vsi = &pf->vsi;
2170         u32             reg;
2171         u16             vector = 1;
2172
2173         /* First set up the adminq - vector 0 */
2174         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2175         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2176
2177         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2178             I40E_PFINT_ICR0_ENA_GRST_MASK |
2179             I40E_PFINT_ICR0_HMC_ERR_MASK |
2180             I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2181             I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2182             I40E_PFINT_ICR0_ENA_VFLR_MASK |
2183             I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2184         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2185
2186         wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2187         wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E);
2188
2189         wr32(hw, I40E_PFINT_DYN_CTL0,
2190             I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2191             I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2192
2193         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2194
2195         /* Next configure the queues */
2196         for (int i = 0; i < vsi->num_queues; i++, vector++) {
2197                 wr32(hw, I40E_PFINT_DYN_CTLN(i), i);
2198                 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2199
2200                 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2201                 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2202                 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2203                 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2204                 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2205                 wr32(hw, I40E_QINT_RQCTL(i), reg);
2206
2207                 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2208                 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2209                 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2210                 ((i+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2211                 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2212                 if (i == (vsi->num_queues - 1))
2213                         reg |= (IXL_QUEUE_EOL
2214                             << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2215                 wr32(hw, I40E_QINT_TQCTL(i), reg);
2216         }
2217 }
2218
2219 /*
2220  * Configure for MSI single vector operation
2221  */
2222 static void
2223 ixl_configure_legacy(struct ixl_pf *pf)
2224 {
2225         struct i40e_hw  *hw = &pf->hw;
2226         u32             reg;
2227
2228
2229         wr32(hw, I40E_PFINT_ITR0(0), 0);
2230         wr32(hw, I40E_PFINT_ITR0(1), 0);
2231
2232
2233         /* Setup "other" causes */
2234         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2235             | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2236             | I40E_PFINT_ICR0_ENA_GRST_MASK
2237             | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2238             | I40E_PFINT_ICR0_ENA_GPIO_MASK
2239             | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2240             | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2241             | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2242             | I40E_PFINT_ICR0_ENA_VFLR_MASK
2243             | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2244             ;
2245         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2246
2247         /* SW_ITR_IDX = 0, but don't change INTENA */
2248         wr32(hw, I40E_PFINT_DYN_CTL0,
2249             I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2250             I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2251         /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2252         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2253
2254         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2255         wr32(hw, I40E_PFINT_LNKLST0, 0);
2256
2257         /* Associate the queue pair to the vector and enable the q int */
2258         reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2259             | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2260             | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2261         wr32(hw, I40E_QINT_RQCTL(0), reg);
2262
2263         reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2264             | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2265             | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2266         wr32(hw, I40E_QINT_TQCTL(0), reg);
2267
2268         /* Next enable the queue pair */
2269         reg = rd32(hw, I40E_QTX_ENA(0));
2270         reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2271         wr32(hw, I40E_QTX_ENA(0), reg);
2272
2273         reg = rd32(hw, I40E_QRX_ENA(0));
2274         reg |= I40E_QRX_ENA_QENA_REQ_MASK;
2275         wr32(hw, I40E_QRX_ENA(0), reg);
2276 }
2277
2278
2279 /*
2280  * Set the Initial ITR state
2281  */
2282 static void
2283 ixl_configure_itr(struct ixl_pf *pf)
2284 {
2285         struct i40e_hw          *hw = &pf->hw;
2286         struct ixl_vsi          *vsi = &pf->vsi;
2287         struct ixl_queue        *que = vsi->queues;
2288
2289         vsi->rx_itr_setting = ixl_rx_itr;
2290         if (ixl_dynamic_rx_itr)
2291                 vsi->rx_itr_setting |= IXL_ITR_DYNAMIC;
2292         vsi->tx_itr_setting = ixl_tx_itr;
2293         if (ixl_dynamic_tx_itr)
2294                 vsi->tx_itr_setting |= IXL_ITR_DYNAMIC;
2295
2296         for (int i = 0; i < vsi->num_queues; i++, que++) {
2297                 struct tx_ring  *txr = &que->txr;
2298                 struct rx_ring  *rxr = &que->rxr;
2299
2300                 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2301                     vsi->rx_itr_setting);
2302                 rxr->itr = vsi->rx_itr_setting;
2303                 rxr->latency = IXL_AVE_LATENCY;
2304                 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2305                     vsi->tx_itr_setting);
2306                 txr->itr = vsi->tx_itr_setting;
2307                 txr->latency = IXL_AVE_LATENCY;
2308         }
2309 }
2310
2311
2312 static int
2313 ixl_allocate_pci_resources(struct ixl_pf *pf)
2314 {
2315         int             rid;
2316         device_t        dev = pf->dev;
2317
2318         rid = PCIR_BAR(0);
2319         pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2320             &rid, RF_ACTIVE);
2321
2322         if (!(pf->pci_mem)) {
2323                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2324                 return (ENXIO);
2325         }
2326
2327         pf->osdep.mem_bus_space_tag =
2328                 rman_get_bustag(pf->pci_mem);
2329         pf->osdep.mem_bus_space_handle =
2330                 rman_get_bushandle(pf->pci_mem);
2331         pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2332         pf->osdep.flush_reg = I40E_GLGEN_STAT;
2333         pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2334
2335         pf->hw.back = &pf->osdep;
2336
2337         /*
2338         ** Now setup MSI or MSI/X, should
2339         ** return us the number of supported
2340         ** vectors. (Will be 1 for MSI)
2341         */
2342         pf->msix = ixl_init_msix(pf);
2343         return (0);
2344 }
2345
2346 static void
2347 ixl_free_pci_resources(struct ixl_pf * pf)
2348 {
2349         struct ixl_vsi          *vsi = &pf->vsi;
2350         struct ixl_queue        *que = vsi->queues;
2351         device_t                dev = pf->dev;
2352         int                     rid, memrid;
2353
2354         memrid = PCIR_BAR(IXL_BAR);
2355
2356         /* We may get here before stations are setup */
2357         if ((!ixl_enable_msix) || (que == NULL))
2358                 goto early;
2359
2360         /*
2361         **  Release all msix VSI resources:
2362         */
2363         for (int i = 0; i < vsi->num_queues; i++, que++) {
2364                 rid = que->msix + 1;
2365                 if (que->tag != NULL) {
2366                         bus_teardown_intr(dev, que->res, que->tag);
2367                         que->tag = NULL;
2368                 }
2369                 if (que->res != NULL)
2370                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2371         }
2372
2373 early:
2374         /* Clean the AdminQ interrupt last */
2375         if (pf->admvec) /* we are doing MSIX */
2376                 rid = pf->admvec + 1;
2377         else
2378                 (pf->msix != 0) ? (rid = 1):(rid = 0);
2379
2380         if (pf->tag != NULL) {
2381                 bus_teardown_intr(dev, pf->res, pf->tag);
2382                 pf->tag = NULL;
2383         }
2384         if (pf->res != NULL)
2385                 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2386
2387         if (pf->msix)
2388                 pci_release_msi(dev);
2389
2390         if (pf->msix_mem != NULL)
2391                 bus_release_resource(dev, SYS_RES_MEMORY,
2392                     memrid, pf->msix_mem);
2393
2394         if (pf->pci_mem != NULL)
2395                 bus_release_resource(dev, SYS_RES_MEMORY,
2396                     PCIR_BAR(0), pf->pci_mem);
2397
2398         return;
2399 }
2400
2401 static void
2402 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2403 {
2404         /* Display supported media types */
2405         if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2406                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2407
2408         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2409                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2410         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2411                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2412         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2413                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2414
2415         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2416             phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4) ||
2417             phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR) ||
2418             phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2419             phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2420             phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2421             phy_type & (1 << I40E_PHY_TYPE_SFI) ||
2422             phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2423                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2424
2425         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2426                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2427         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2428                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2429         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2430                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2431
2432         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2433             phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2434             phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2435             phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2436             phy_type & (1 << I40E_PHY_TYPE_XLPPI) ||
2437             /* KR4 uses CR4 until the OS has the real media type */
2438             phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2439                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2440
2441         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2442                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2443         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2444                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2445 }
2446
2447 /*********************************************************************
2448  *
2449  *  Setup networking device structure and register an interface.
2450  *
2451  **********************************************************************/
2452 static int
2453 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2454 {
2455         struct ifnet            *ifp;
2456         struct i40e_hw          *hw = vsi->hw;
2457         struct ixl_queue        *que = vsi->queues;
2458         struct i40e_aq_get_phy_abilities_resp abilities;
2459         enum i40e_status_code aq_error = 0;
2460
2461         INIT_DEBUGOUT("ixl_setup_interface: begin");
2462
2463         ifp = vsi->ifp = if_alloc(IFT_ETHER);
2464         if (ifp == NULL) {
2465                 device_printf(dev, "can not allocate ifnet structure\n");
2466                 return (-1);
2467         }
2468         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2469         ifp->if_mtu = ETHERMTU;
2470         if_initbaudrate(ifp, IF_Gbps(40));
2471         ifp->if_init = ixl_init;
2472         ifp->if_softc = vsi;
2473         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2474         ifp->if_ioctl = ixl_ioctl;
2475
2476 #if __FreeBSD_version >= 1100036
2477         if_setgetcounterfn(ifp, ixl_get_counter);
2478 #endif
2479
2480         ifp->if_transmit = ixl_mq_start;
2481
2482         ifp->if_qflush = ixl_qflush;
2483
2484         ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2485
2486         vsi->max_frame_size =
2487             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2488             + ETHER_VLAN_ENCAP_LEN;
2489
2490         /*
2491          * Tell the upper layer(s) we support long frames.
2492          */
2493         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2494
2495         ifp->if_capabilities |= IFCAP_HWCSUM;
2496         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2497         ifp->if_capabilities |= IFCAP_TSO;
2498         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2499         ifp->if_capabilities |= IFCAP_LRO;
2500
2501         /* VLAN capabilties */
2502         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2503                              |  IFCAP_VLAN_HWTSO
2504                              |  IFCAP_VLAN_MTU
2505                              |  IFCAP_VLAN_HWCSUM;
2506         ifp->if_capenable = ifp->if_capabilities;
2507
2508         /*
2509         ** Don't turn this on by default, if vlans are
2510         ** created on another pseudo device (eg. lagg)
2511         ** then vlan events are not passed thru, breaking
2512         ** operation, but with HW FILTER off it works. If
2513         ** using vlans directly on the ixl driver you can
2514         ** enable this and get full hardware tag filtering.
2515         */
2516         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2517
2518         /*
2519          * Specify the media types supported by this adapter and register
2520          * callbacks to update media and link information
2521          */
2522         ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2523                      ixl_media_status);
2524
2525         aq_error = i40e_aq_get_phy_capabilities(hw,
2526             FALSE, TRUE, &abilities, NULL);
2527         /* May need delay to detect fiber correctly */
2528         if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2529                 i40e_msec_delay(200);
2530                 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2531                     TRUE, &abilities, NULL);
2532         }
2533         if (aq_error) {
2534                 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2535                         device_printf(dev, "Unknown PHY type detected!\n");
2536                 else
2537                         device_printf(dev,
2538                             "Error getting supported media types, err %d,"
2539                             " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2540                 return (0);
2541         }
2542
2543         ixl_add_ifmedia(vsi, abilities.phy_type);
2544
2545         /* Use autoselect media by default */
2546         ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2547         ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2548
2549         ether_ifattach(ifp, hw->mac.addr);
2550
2551         return (0);
2552 }
2553
2554 /*
2555 ** Run when the Admin Queue gets a
2556 ** link transition interrupt.
2557 */
2558 static void
2559 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
2560 {
2561         struct i40e_hw  *hw = &pf->hw;
2562         struct i40e_aqc_get_link_status *status =
2563             (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
2564         bool check;
2565
2566         hw->phy.get_link_info = TRUE;
2567         check = i40e_get_link_status(hw);
2568         pf->link_up = check;
2569 #ifdef IXL_DEBUG
2570         printf("Link is %s\n", check ? "up":"down");
2571 #endif
2572         /* Report if Unqualified modules are found */
2573         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
2574             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
2575             (!(status->link_info & I40E_AQ_LINK_UP)))
2576                 device_printf(pf->dev, "Link failed because "
2577                     "an unqualified module was detected\n");
2578
2579         return;
2580 }
2581
2582 /*********************************************************************
2583  *
2584  *  Get Firmware Switch configuration
2585  *      - this will need to be more robust when more complex
2586  *        switch configurations are enabled.
2587  *
2588  **********************************************************************/
2589 static int
2590 ixl_switch_config(struct ixl_pf *pf)
2591 {
2592         struct i40e_hw  *hw = &pf->hw;
2593         struct ixl_vsi  *vsi = &pf->vsi;
2594         device_t        dev = vsi->dev;
2595         struct i40e_aqc_get_switch_config_resp *sw_config;
2596         u8      aq_buf[I40E_AQ_LARGE_BUF];
2597         int     ret;
2598         u16     next = 0;
2599
2600         memset(&aq_buf, 0, sizeof(aq_buf));
2601         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
2602         ret = i40e_aq_get_switch_config(hw, sw_config,
2603             sizeof(aq_buf), &next, NULL);
2604         if (ret) {
2605                 device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n",
2606                     ret);
2607                 return (ret);
2608         }
2609 #ifdef IXL_DEBUG
2610         device_printf(dev,
2611             "Switch config: header reported: %d in structure, %d total\n",
2612             sw_config->header.num_reported, sw_config->header.num_total);
2613         for (int i = 0; i < sw_config->header.num_reported; i++) {
2614                 device_printf(dev,
2615                     "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
2616                     sw_config->element[i].element_type,
2617                     sw_config->element[i].seid,
2618                     sw_config->element[i].uplink_seid,
2619                     sw_config->element[i].downlink_seid);
2620         }
2621 #endif
2622         /* Simplified due to a single VSI at the moment */
2623         vsi->uplink_seid = sw_config->element[0].uplink_seid;
2624         vsi->downlink_seid = sw_config->element[0].downlink_seid;
2625         vsi->seid = sw_config->element[0].seid;
2626         return (ret);
2627 }
2628
2629 /*********************************************************************
2630  *
2631  *  Initialize the VSI:  this handles contexts, which means things
2632  *                       like the number of descriptors, buffer size,
2633  *                       plus we init the rings thru this function.
2634  *
2635  **********************************************************************/
2636 static int
2637 ixl_initialize_vsi(struct ixl_vsi *vsi)
2638 {
2639         struct ixl_pf           *pf = vsi->back;
2640         struct ixl_queue        *que = vsi->queues;
2641         device_t                dev = vsi->dev;
2642         struct i40e_hw          *hw = vsi->hw;
2643         struct i40e_vsi_context ctxt;
2644         int                     err = 0;
2645
2646         memset(&ctxt, 0, sizeof(ctxt));
2647         ctxt.seid = vsi->seid;
2648         if (pf->veb_seid != 0)
2649                 ctxt.uplink_seid = pf->veb_seid;
2650         ctxt.pf_num = hw->pf_id;
2651         err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2652         if (err) {
2653                 device_printf(dev,"get vsi params failed %x!!\n", err);
2654                 return (err);
2655         }
2656 #ifdef IXL_DEBUG
2657         printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
2658             "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
2659             "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
2660             ctxt.uplink_seid, ctxt.vsi_number,
2661             ctxt.vsis_allocated, ctxt.vsis_unallocated,
2662             ctxt.flags, ctxt.pf_num, ctxt.vf_num,
2663             ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
2664 #endif
2665         /*
2666         ** Set the queue and traffic class bits
2667         **  - when multiple traffic classes are supported
2668         **    this will need to be more robust.
2669         */
2670         ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2671         ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
2672         ctxt.info.queue_mapping[0] = 0;
2673         ctxt.info.tc_mapping[0] = 0x0800;
2674
2675         /* Set VLAN receive stripping mode */
2676         ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
2677         ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
2678         if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
2679             ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2680         else
2681             ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2682
2683         /* Keep copy of VSI info in VSI for statistic counters */
2684         memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2685
2686         /* Reset VSI statistics */
2687         ixl_vsi_reset_stats(vsi);
2688         vsi->hw_filters_add = 0;
2689         vsi->hw_filters_del = 0;
2690
2691         ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
2692
2693         err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2694         if (err) {
2695                 device_printf(dev,"update vsi params failed %x!!\n",
2696                    hw->aq.asq_last_status);
2697                 return (err);
2698         }
2699
2700         for (int i = 0; i < vsi->num_queues; i++, que++) {
2701                 struct tx_ring          *txr = &que->txr;
2702                 struct rx_ring          *rxr = &que->rxr;
2703                 struct i40e_hmc_obj_txq tctx;
2704                 struct i40e_hmc_obj_rxq rctx;
2705                 u32                     txctl;
2706                 u16                     size;
2707
2708
2709                 /* Setup the HMC TX Context  */
2710                 size = que->num_desc * sizeof(struct i40e_tx_desc);
2711                 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
2712                 tctx.new_context = 1;
2713                 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
2714                 tctx.qlen = que->num_desc;
2715                 tctx.fc_ena = 0;
2716                 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
2717                 /* Enable HEAD writeback */
2718                 tctx.head_wb_ena = 1;
2719                 tctx.head_wb_addr = txr->dma.pa +
2720                     (que->num_desc * sizeof(struct i40e_tx_desc));
2721                 tctx.rdylist_act = 0;
2722                 err = i40e_clear_lan_tx_queue_context(hw, i);
2723                 if (err) {
2724                         device_printf(dev, "Unable to clear TX context\n");
2725                         break;
2726                 }
2727                 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
2728                 if (err) {
2729                         device_printf(dev, "Unable to set TX context\n");
2730                         break;
2731                 }
2732                 /* Associate the ring with this PF */
2733                 txctl = I40E_QTX_CTL_PF_QUEUE;
2734                 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2735                     I40E_QTX_CTL_PF_INDX_MASK);
2736                 wr32(hw, I40E_QTX_CTL(i), txctl);
2737                 ixl_flush(hw);
2738
2739                 /* Do ring (re)init */
2740                 ixl_init_tx_ring(que);
2741
2742                 /* Next setup the HMC RX Context  */
2743                 if (vsi->max_frame_size <= MCLBYTES)
2744                         rxr->mbuf_sz = MCLBYTES;
2745                 else
2746                         rxr->mbuf_sz = MJUMPAGESIZE;
2747
2748                 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
2749
2750                 /* Set up an RX context for the HMC */
2751                 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2752                 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
2753                 /* ignore header split for now */
2754                 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
2755                 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
2756                     vsi->max_frame_size : max_rxmax;
2757                 rctx.dtype = 0;
2758                 rctx.dsize = 1; /* do 32byte descriptors */
2759                 rctx.hsplit_0 = 0;  /* no HDR split initially */
2760                 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
2761                 rctx.qlen = que->num_desc;
2762                 rctx.tphrdesc_ena = 1;
2763                 rctx.tphwdesc_ena = 1;
2764                 rctx.tphdata_ena = 0;
2765                 rctx.tphhead_ena = 0;
2766                 rctx.lrxqthresh = 2;
2767                 rctx.crcstrip = 1;
2768                 rctx.l2tsel = 1;
2769                 rctx.showiv = 1;
2770                 rctx.fc_ena = 0;
2771                 rctx.prefena = 1;
2772
2773                 err = i40e_clear_lan_rx_queue_context(hw, i);
2774                 if (err) {
2775                         device_printf(dev,
2776                             "Unable to clear RX context %d\n", i);
2777                         break;
2778                 }
2779                 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
2780                 if (err) {
2781                         device_printf(dev, "Unable to set RX context %d\n", i);
2782                         break;
2783                 }
2784                 err = ixl_init_rx_ring(que);
2785                 if (err) {
2786                         device_printf(dev, "Fail in init_rx_ring %d\n", i);
2787                         break;
2788                 }
2789                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), 0);
2790 #ifdef DEV_NETMAP
2791                 /* preserve queue */
2792                 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
2793                         struct netmap_adapter *na = NA(vsi->ifp);
2794                         struct netmap_kring *kring = &na->rx_rings[i];
2795                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
2796                         wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
2797                 } else
2798 #endif /* DEV_NETMAP */
2799                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
2800         }
2801         return (err);
2802 }
2803
2804
2805 /*********************************************************************
2806  *
2807  *  Free all VSI structs.
2808  *
2809  **********************************************************************/
2810 void
2811 ixl_free_vsi(struct ixl_vsi *vsi)
2812 {
2813         struct ixl_pf           *pf = (struct ixl_pf *)vsi->back;
2814         struct ixl_queue        *que = vsi->queues;
2815
2816         /* Free station queues */
2817         for (int i = 0; i < vsi->num_queues; i++, que++) {
2818                 struct tx_ring *txr = &que->txr;
2819                 struct rx_ring *rxr = &que->rxr;
2820
2821                 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
2822                         continue;
2823                 IXL_TX_LOCK(txr);
2824                 ixl_free_que_tx(que);
2825                 if (txr->base)
2826                         i40e_free_dma_mem(&pf->hw, &txr->dma);
2827                 IXL_TX_UNLOCK(txr);
2828                 IXL_TX_LOCK_DESTROY(txr);
2829
2830                 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
2831                         continue;
2832                 IXL_RX_LOCK(rxr);
2833                 ixl_free_que_rx(que);
2834                 if (rxr->base)
2835                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
2836                 IXL_RX_UNLOCK(rxr);
2837                 IXL_RX_LOCK_DESTROY(rxr);
2838
2839         }
2840         free(vsi->queues, M_DEVBUF);
2841
2842         /* Free VSI filter list */
2843         ixl_free_mac_filters(vsi);
2844 }
2845
2846 static void
2847 ixl_free_mac_filters(struct ixl_vsi *vsi)
2848 {
2849         struct ixl_mac_filter *f;
2850
2851         while (!SLIST_EMPTY(&vsi->ftl)) {
2852                 f = SLIST_FIRST(&vsi->ftl);
2853                 SLIST_REMOVE_HEAD(&vsi->ftl, next);
2854                 free(f, M_DEVBUF);
2855         }
2856 }
2857
2858
2859 /*********************************************************************
2860  *
2861  *  Allocate memory for the VSI (virtual station interface) and their
2862  *  associated queues, rings and the descriptors associated with each,
2863  *  called only once at attach.
2864  *
2865  **********************************************************************/
2866 static int
2867 ixl_setup_stations(struct ixl_pf *pf)
2868 {
2869         device_t                dev = pf->dev;
2870         struct ixl_vsi          *vsi;
2871         struct ixl_queue        *que;
2872         struct tx_ring          *txr;
2873         struct rx_ring          *rxr;
2874         int                     rsize, tsize;
2875         int                     error = I40E_SUCCESS;
2876
2877         vsi = &pf->vsi;
2878         vsi->back = (void *)pf;
2879         vsi->hw = &pf->hw;
2880         vsi->id = 0;
2881         vsi->num_vlans = 0;
2882         vsi->back = pf;
2883
2884         /* Get memory for the station queues */
2885         if (!(vsi->queues =
2886             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
2887             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2888                 device_printf(dev, "Unable to allocate queue memory\n");
2889                 error = ENOMEM;
2890                 goto early;
2891         }
2892
2893         for (int i = 0; i < vsi->num_queues; i++) {
2894                 que = &vsi->queues[i];
2895                 que->num_desc = ixl_ringsz;
2896                 que->me = i;
2897                 que->vsi = vsi;
2898                 /* mark the queue as active */
2899                 vsi->active_queues |= (u64)1 << que->me;
2900                 txr = &que->txr;
2901                 txr->que = que;
2902                 txr->tail = I40E_QTX_TAIL(que->me);
2903
2904                 /* Initialize the TX lock */
2905                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2906                     device_get_nameunit(dev), que->me);
2907                 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
2908                 /* Create the TX descriptor ring */
2909                 tsize = roundup2((que->num_desc *
2910                     sizeof(struct i40e_tx_desc)) +
2911                     sizeof(u32), DBA_ALIGN);
2912                 if (i40e_allocate_dma_mem(&pf->hw,
2913                     &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
2914                         device_printf(dev,
2915                             "Unable to allocate TX Descriptor memory\n");
2916                         error = ENOMEM;
2917                         goto fail;
2918                 }
2919                 txr->base = (struct i40e_tx_desc *)txr->dma.va;
2920                 bzero((void *)txr->base, tsize);
2921                 /* Now allocate transmit soft structs for the ring */
2922                 if (ixl_allocate_tx_data(que)) {
2923                         device_printf(dev,
2924                             "Critical Failure setting up TX structures\n");
2925                         error = ENOMEM;
2926                         goto fail;
2927                 }
2928                 /* Allocate a buf ring */
2929                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
2930                     M_WAITOK, &txr->mtx);
2931                 if (txr->br == NULL) {
2932                         device_printf(dev,
2933                             "Critical Failure setting up TX buf ring\n");
2934                         error = ENOMEM;
2935                         goto fail;
2936                 }
2937
2938                 /*
2939                  * Next the RX queues...
2940                  */
2941                 rsize = roundup2(que->num_desc *
2942                     sizeof(union i40e_rx_desc), DBA_ALIGN);
2943                 rxr = &que->rxr;
2944                 rxr->que = que;
2945                 rxr->tail = I40E_QRX_TAIL(que->me);
2946
2947                 /* Initialize the RX side lock */
2948                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2949                     device_get_nameunit(dev), que->me);
2950                 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
2951
2952                 if (i40e_allocate_dma_mem(&pf->hw,
2953                     &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
2954                         device_printf(dev,
2955                             "Unable to allocate RX Descriptor memory\n");
2956                         error = ENOMEM;
2957                         goto fail;
2958                 }
2959                 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
2960                 bzero((void *)rxr->base, rsize);
2961
2962                 /* Allocate receive soft structs for the ring*/
2963                 if (ixl_allocate_rx_data(que)) {
2964                         device_printf(dev,
2965                             "Critical Failure setting up receive structs\n");
2966                         error = ENOMEM;
2967                         goto fail;
2968                 }
2969         }
2970
2971         return (0);
2972
2973 fail:
2974         for (int i = 0; i < vsi->num_queues; i++) {
2975                 que = &vsi->queues[i];
2976                 rxr = &que->rxr;
2977                 txr = &que->txr;
2978                 if (rxr->base)
2979                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
2980                 if (txr->base)
2981                         i40e_free_dma_mem(&pf->hw, &txr->dma);
2982         }
2983
2984 early:
2985         return (error);
2986 }
2987
2988 /*
2989 ** Provide a update to the queue RX
2990 ** interrupt moderation value.
2991 */
2992 static void
2993 ixl_set_queue_rx_itr(struct ixl_queue *que)
2994 {
2995         struct ixl_vsi  *vsi = que->vsi;
2996         struct i40e_hw  *hw = vsi->hw;
2997         struct rx_ring  *rxr = &que->rxr;
2998         u16             rx_itr;
2999         u16             rx_latency = 0;
3000         int             rx_bytes;
3001
3002
3003         /* Idle, do nothing */
3004         if (rxr->bytes == 0)
3005                 return;
3006
3007         if (ixl_dynamic_rx_itr) {
3008                 rx_bytes = rxr->bytes/rxr->itr;
3009                 rx_itr = rxr->itr;
3010
3011                 /* Adjust latency range */
3012                 switch (rxr->latency) {
3013                 case IXL_LOW_LATENCY:
3014                         if (rx_bytes > 10) {
3015                                 rx_latency = IXL_AVE_LATENCY;
3016                                 rx_itr = IXL_ITR_20K;
3017                         }
3018                         break;
3019                 case IXL_AVE_LATENCY:
3020                         if (rx_bytes > 20) {
3021                                 rx_latency = IXL_BULK_LATENCY;
3022                                 rx_itr = IXL_ITR_8K;
3023                         } else if (rx_bytes <= 10) {
3024                                 rx_latency = IXL_LOW_LATENCY;
3025                                 rx_itr = IXL_ITR_100K;
3026                         }
3027                         break;
3028                 case IXL_BULK_LATENCY:
3029                         if (rx_bytes <= 20) {
3030                                 rx_latency = IXL_AVE_LATENCY;
3031                                 rx_itr = IXL_ITR_20K;
3032                         }
3033                         break;
3034                  }
3035
3036                 rxr->latency = rx_latency;
3037
3038                 if (rx_itr != rxr->itr) {
3039                         /* do an exponential smoothing */
3040                         rx_itr = (10 * rx_itr * rxr->itr) /
3041                             ((9 * rx_itr) + rxr->itr);
3042                         rxr->itr = rx_itr & IXL_MAX_ITR;
3043                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3044                             que->me), rxr->itr);
3045                 }
3046         } else { /* We may have have toggled to non-dynamic */
3047                 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3048                         vsi->rx_itr_setting = ixl_rx_itr;
3049                 /* Update the hardware if needed */
3050                 if (rxr->itr != vsi->rx_itr_setting) {
3051                         rxr->itr = vsi->rx_itr_setting;
3052                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3053                             que->me), rxr->itr);
3054                 }
3055         }
3056         rxr->bytes = 0;
3057         rxr->packets = 0;
3058         return;
3059 }
3060
3061
3062 /*
3063 ** Provide a update to the queue TX
3064 ** interrupt moderation value.
3065 */
3066 static void
3067 ixl_set_queue_tx_itr(struct ixl_queue *que)
3068 {
3069         struct ixl_vsi  *vsi = que->vsi;
3070         struct i40e_hw  *hw = vsi->hw;
3071         struct tx_ring  *txr = &que->txr;
3072         u16             tx_itr;
3073         u16             tx_latency = 0;
3074         int             tx_bytes;
3075
3076
3077         /* Idle, do nothing */
3078         if (txr->bytes == 0)
3079                 return;
3080
3081         if (ixl_dynamic_tx_itr) {
3082                 tx_bytes = txr->bytes/txr->itr;
3083                 tx_itr = txr->itr;
3084
3085                 switch (txr->latency) {
3086                 case IXL_LOW_LATENCY:
3087                         if (tx_bytes > 10) {
3088                                 tx_latency = IXL_AVE_LATENCY;
3089                                 tx_itr = IXL_ITR_20K;
3090                         }
3091                         break;
3092                 case IXL_AVE_LATENCY:
3093                         if (tx_bytes > 20) {
3094                                 tx_latency = IXL_BULK_LATENCY;
3095                                 tx_itr = IXL_ITR_8K;
3096                         } else if (tx_bytes <= 10) {
3097                                 tx_latency = IXL_LOW_LATENCY;
3098                                 tx_itr = IXL_ITR_100K;
3099                         }
3100                         break;
3101                 case IXL_BULK_LATENCY:
3102                         if (tx_bytes <= 20) {
3103                                 tx_latency = IXL_AVE_LATENCY;
3104                                 tx_itr = IXL_ITR_20K;
3105                         }
3106                         break;
3107                 }
3108
3109                 txr->latency = tx_latency;
3110
3111                 if (tx_itr != txr->itr) {
3112                  /* do an exponential smoothing */
3113                         tx_itr = (10 * tx_itr * txr->itr) /
3114                             ((9 * tx_itr) + txr->itr);
3115                         txr->itr = tx_itr & IXL_MAX_ITR;
3116                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3117                             que->me), txr->itr);
3118                 }
3119
3120         } else { /* We may have have toggled to non-dynamic */
3121                 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3122                         vsi->tx_itr_setting = ixl_tx_itr;
3123                 /* Update the hardware if needed */
3124                 if (txr->itr != vsi->tx_itr_setting) {
3125                         txr->itr = vsi->tx_itr_setting;
3126                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3127                             que->me), txr->itr);
3128                 }
3129         }
3130         txr->bytes = 0;
3131         txr->packets = 0;
3132         return;
3133 }
3134
3135 #define QUEUE_NAME_LEN 32
3136
3137 static void
3138 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3139     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3140 {
3141         struct sysctl_oid *tree;
3142         struct sysctl_oid_list *child;
3143         struct sysctl_oid_list *vsi_list;
3144
3145         tree = device_get_sysctl_tree(pf->dev);
3146         child = SYSCTL_CHILDREN(tree);
3147         vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3148                                    CTLFLAG_RD, NULL, "VSI Number");
3149         vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3150
3151         ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3152 }
3153
3154 static void
3155 ixl_add_hw_stats(struct ixl_pf *pf)
3156 {
3157         device_t dev = pf->dev;
3158         struct ixl_vsi *vsi = &pf->vsi;
3159         struct ixl_queue *queues = vsi->queues;
3160         struct i40e_hw_port_stats *pf_stats = &pf->stats;
3161
3162         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3163         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3164         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3165         struct sysctl_oid_list *vsi_list;
3166
3167         struct sysctl_oid *queue_node;
3168         struct sysctl_oid_list *queue_list;
3169
3170         struct tx_ring *txr;
3171         struct rx_ring *rxr;
3172         char queue_namebuf[QUEUE_NAME_LEN];
3173
3174         /* Driver statistics */
3175         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3176                         CTLFLAG_RD, &pf->watchdog_events,
3177                         "Watchdog timeouts");
3178         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3179                         CTLFLAG_RD, &pf->admin_irq,
3180                         "Admin Queue IRQ Handled");
3181
3182         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "vc_debug_level",
3183             CTLFLAG_RW, &pf->vc_debug_lvl, 0,
3184             "PF/VF Virtual Channel debug logging level");
3185
3186         ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3187         vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3188
3189         /* Queue statistics */
3190         for (int q = 0; q < vsi->num_queues; q++) {
3191                 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3192                 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3193                     OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3194                 queue_list = SYSCTL_CHILDREN(queue_node);
3195
3196                 txr = &(queues[q].txr);
3197                 rxr = &(queues[q].rxr);
3198
3199                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3200                                 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3201                                 "m_defrag() failed");
3202                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped",
3203                                 CTLFLAG_RD, &(queues[q].dropped_pkts),
3204                                 "Driver dropped packets");
3205                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3206                                 CTLFLAG_RD, &(queues[q].irqs),
3207                                 "irqs on this queue");
3208                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3209                                 CTLFLAG_RD, &(queues[q].tso),
3210                                 "TSO");
3211                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3212                                 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3213                                 "Driver tx dma failure in xmit");
3214                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3215                                 CTLFLAG_RD, &(txr->no_desc),
3216                                 "Queue No Descriptor Available");
3217                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3218                                 CTLFLAG_RD, &(txr->total_packets),
3219                                 "Queue Packets Transmitted");
3220                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3221                                 CTLFLAG_RD, &(txr->tx_bytes),
3222                                 "Queue Bytes Transmitted");
3223                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3224                                 CTLFLAG_RD, &(rxr->rx_packets),
3225                                 "Queue Packets Received");
3226                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3227                                 CTLFLAG_RD, &(rxr->rx_bytes),
3228                                 "Queue Bytes Received");
3229         }
3230
3231         /* MAC stats */
3232         ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3233 }
3234
3235 static void
3236 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3237         struct sysctl_oid_list *child,
3238         struct i40e_eth_stats *eth_stats)
3239 {
3240         struct ixl_sysctl_info ctls[] =
3241         {
3242                 {&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3243                 {&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3244                         "Unicast Packets Received"},
3245                 {&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3246                         "Multicast Packets Received"},
3247                 {&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3248                         "Broadcast Packets Received"},
3249                 {&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3250                 {&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3251                 {&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3252                 {&eth_stats->tx_multicast, "mcast_pkts_txd",
3253                         "Multicast Packets Transmitted"},
3254                 {&eth_stats->tx_broadcast, "bcast_pkts_txd",
3255                         "Broadcast Packets Transmitted"},
3256                 // end
3257                 {0,0,0}
3258         };
3259
3260         struct ixl_sysctl_info *entry = ctls;
3261         while (entry->stat != 0)
3262         {
3263                 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3264                                 CTLFLAG_RD, entry->stat,
3265                                 entry->description);
3266                 entry++;
3267         }
3268 }
3269
3270 static void
3271 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3272         struct sysctl_oid_list *child,
3273         struct i40e_hw_port_stats *stats)
3274 {
3275         struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3276                                     CTLFLAG_RD, NULL, "Mac Statistics");
3277         struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3278
3279         struct i40e_eth_stats *eth_stats = &stats->eth;
3280         ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3281
3282         struct ixl_sysctl_info ctls[] =
3283         {
3284                 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3285                 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3286                 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3287                 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3288                 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3289                 /* Packet Reception Stats */
3290                 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3291                 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3292                 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3293                 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3294                 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3295                 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3296                 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3297                 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3298                 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3299                 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3300                 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3301                 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3302                 /* Packet Transmission Stats */
3303                 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3304                 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3305                 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3306                 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3307                 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3308                 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3309                 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3310                 /* Flow control */
3311                 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3312                 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3313                 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3314                 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3315                 /* End */
3316                 {0,0,0}
3317         };
3318
3319         struct ixl_sysctl_info *entry = ctls;
3320         while (entry->stat != 0)
3321         {
3322                 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3323                                 CTLFLAG_RD, entry->stat,
3324                                 entry->description);
3325                 entry++;
3326         }
3327 }
3328
3329 /*
3330 ** ixl_config_rss - setup RSS
3331 **  - note this is done for the single vsi
3332 */
3333 static void ixl_config_rss(struct ixl_vsi *vsi)
3334 {
3335         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3336         struct i40e_hw  *hw = vsi->hw;
3337         u32             lut = 0;
3338         u64             set_hena = 0, hena;
3339         int             i, j, que_id;
3340 #ifdef RSS
3341         u32             rss_hash_config;
3342         u32             rss_seed[IXL_KEYSZ];
3343 #else
3344         u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3345                             0x183cfd8c, 0xce880440, 0x580cbc3c,
3346                             0x35897377, 0x328b25e1, 0x4fa98922,
3347                             0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3348 #endif
3349
3350 #ifdef RSS
3351         /* Fetch the configured RSS key */
3352         rss_getkey((uint8_t *) &rss_seed);
3353 #endif
3354
3355         /* Fill out hash function seed */
3356         for (i = 0; i < IXL_KEYSZ; i++)
3357                 wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3358
3359         /* Enable PCTYPES for RSS: */
3360 #ifdef RSS
3361         rss_hash_config = rss_gethashconfig();
3362         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3363                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3364         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3365                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3366         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3367                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3368         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3369                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3370         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3371                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3372         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3373                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3374         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3375                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3376 #else
3377         set_hena =
3378                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3379                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3380                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3381                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3382                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3383                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3384                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3385                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3386                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3387                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3388                 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3389 #endif
3390         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
3391             ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
3392         hena |= set_hena;
3393         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
3394         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3395
3396         /* Populate the LUT with max no. of queues in round robin fashion */
3397         for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3398                 if (j == vsi->num_queues)
3399                         j = 0;
3400 #ifdef RSS
3401                 /*
3402                  * Fetch the RSS bucket id for the given indirection entry.
3403                  * Cap it at the number of configured buckets (which is
3404                  * num_queues.)
3405                  */
3406                 que_id = rss_get_indirection_to_bucket(i);
3407                 que_id = que_id % vsi->num_queues;
3408 #else
3409                 que_id = j;
3410 #endif
3411                 /* lut = 4-byte sliding window of 4 lut entries */
3412                 lut = (lut << 8) | (que_id &
3413                     ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3414                 /* On i = 3, we have 4 entries in lut; write to the register */
3415                 if ((i & 3) == 3)
3416                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3417         }
3418         ixl_flush(hw);
3419 }
3420
3421
3422 /*
3423 ** This routine is run via an vlan config EVENT,
3424 ** it enables us to use the HW Filter table since
3425 ** we can get the vlan id. This just creates the
3426 ** entry in the soft version of the VFTA, init will
3427 ** repopulate the real table.
3428 */
3429 static void
3430 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3431 {
3432         struct ixl_vsi  *vsi = ifp->if_softc;
3433         struct i40e_hw  *hw = vsi->hw;
3434         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3435
3436         if (ifp->if_softc !=  arg)   /* Not our event */
3437                 return;
3438
3439         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3440                 return;
3441
3442         IXL_PF_LOCK(pf);
3443         ++vsi->num_vlans;
3444         ixl_add_filter(vsi, hw->mac.addr, vtag);
3445         IXL_PF_UNLOCK(pf);
3446 }
3447
3448 /*
3449 ** This routine is run via an vlan
3450 ** unconfig EVENT, remove our entry
3451 ** in the soft vfta.
3452 */
3453 static void
3454 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3455 {
3456         struct ixl_vsi  *vsi = ifp->if_softc;
3457         struct i40e_hw  *hw = vsi->hw;
3458         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3459
3460         if (ifp->if_softc !=  arg)
3461                 return;
3462
3463         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3464                 return;
3465
3466         IXL_PF_LOCK(pf);
3467         --vsi->num_vlans;
3468         ixl_del_filter(vsi, hw->mac.addr, vtag);
3469         IXL_PF_UNLOCK(pf);
3470 }
3471
3472 /*
3473 ** This routine updates vlan filters, called by init
3474 ** it scans the filter table and then updates the hw
3475 ** after a soft reset.
3476 */
3477 static void
3478 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
3479 {
3480         struct ixl_mac_filter   *f;
3481         int                     cnt = 0, flags;
3482
3483         if (vsi->num_vlans == 0)
3484                 return;
3485         /*
3486         ** Scan the filter list for vlan entries,
3487         ** mark them for addition and then call
3488         ** for the AQ update.
3489         */
3490         SLIST_FOREACH(f, &vsi->ftl, next) {
3491                 if (f->flags & IXL_FILTER_VLAN) {
3492                         f->flags |=
3493                             (IXL_FILTER_ADD |
3494                             IXL_FILTER_USED);
3495                         cnt++;
3496                 }
3497         }
3498         if (cnt == 0) {
3499                 printf("setup vlan: no filters found!\n");
3500                 return;
3501         }
3502         flags = IXL_FILTER_VLAN;
3503         flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3504         ixl_add_hw_filters(vsi, flags, cnt);
3505         return;
3506 }
3507
3508 /*
3509 ** Initialize filter list and add filters that the hardware
3510 ** needs to know about.
3511 */
3512 static void
3513 ixl_init_filters(struct ixl_vsi *vsi)
3514 {
3515         /* Add broadcast address */
3516         ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
3517 }
3518
3519 /*
3520 ** This routine adds mulicast filters
3521 */
3522 static void
3523 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
3524 {
3525         struct ixl_mac_filter *f;
3526
3527         /* Does one already exist */
3528         f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3529         if (f != NULL)
3530                 return;
3531
3532         f = ixl_get_filter(vsi);
3533         if (f == NULL) {
3534                 printf("WARNING: no filter available!!\n");
3535                 return;
3536         }
3537         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3538         f->vlan = IXL_VLAN_ANY;
3539         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
3540             | IXL_FILTER_MC);
3541
3542         return;
3543 }
3544
3545 static void
3546 ixl_reconfigure_filters(struct ixl_vsi *vsi)
3547 {
3548
3549         ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
3550 }
3551
3552 /*
3553 ** This routine adds macvlan filters
3554 */
3555 static void
3556 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3557 {
3558         struct ixl_mac_filter   *f, *tmp;
3559         struct ixl_pf           *pf;
3560         device_t                dev;
3561
3562         DEBUGOUT("ixl_add_filter: begin");
3563
3564         pf = vsi->back;
3565         dev = pf->dev;
3566
3567         /* Does one already exist */
3568         f = ixl_find_filter(vsi, macaddr, vlan);
3569         if (f != NULL)
3570                 return;
3571         /*
3572         ** Is this the first vlan being registered, if so we
3573         ** need to remove the ANY filter that indicates we are
3574         ** not in a vlan, and replace that with a 0 filter.
3575         */
3576         if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
3577                 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3578                 if (tmp != NULL) {
3579                         ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
3580                         ixl_add_filter(vsi, macaddr, 0);
3581                 }
3582         }
3583
3584         f = ixl_get_filter(vsi);
3585         if (f == NULL) {
3586                 device_printf(dev, "WARNING: no filter available!!\n");
3587                 return;
3588         }
3589         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3590         f->vlan = vlan;
3591         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3592         if (f->vlan != IXL_VLAN_ANY)
3593                 f->flags |= IXL_FILTER_VLAN;
3594         else
3595                 vsi->num_macs++;
3596
3597         ixl_add_hw_filters(vsi, f->flags, 1);
3598         return;
3599 }
3600
3601 static void
3602 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3603 {
3604         struct ixl_mac_filter *f;
3605
3606         f = ixl_find_filter(vsi, macaddr, vlan);
3607         if (f == NULL)
3608                 return;
3609
3610         f->flags |= IXL_FILTER_DEL;
3611         ixl_del_hw_filters(vsi, 1);
3612         vsi->num_macs--;
3613
3614         /* Check if this is the last vlan removal */
3615         if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
3616                 /* Switch back to a non-vlan filter */
3617                 ixl_del_filter(vsi, macaddr, 0);
3618                 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
3619         }
3620         return;
3621 }
3622
3623 /*
3624 ** Find the filter with both matching mac addr and vlan id
3625 */
3626 static struct ixl_mac_filter *
3627 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3628 {
3629         struct ixl_mac_filter   *f;
3630         bool                    match = FALSE;
3631
3632         SLIST_FOREACH(f, &vsi->ftl, next) {
3633                 if (!cmp_etheraddr(f->macaddr, macaddr))
3634                         continue;
3635                 if (f->vlan == vlan) {
3636                         match = TRUE;
3637                         break;
3638                 }
3639         }
3640
3641         if (!match)
3642                 f = NULL;
3643         return (f);
3644 }
3645
3646 /*
3647 ** This routine takes additions to the vsi filter
3648 ** table and creates an Admin Queue call to create
3649 ** the filters in the hardware.
3650 */
3651 static void
3652 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
3653 {
3654         struct i40e_aqc_add_macvlan_element_data *a, *b;
3655         struct ixl_mac_filter   *f;
3656         struct ixl_pf           *pf;
3657         struct i40e_hw          *hw;
3658         device_t                dev;
3659         int                     err, j = 0;
3660
3661         pf = vsi->back;
3662         dev = pf->dev;
3663         hw = &pf->hw;
3664         IXL_PF_LOCK_ASSERT(pf);
3665
3666         a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
3667             M_DEVBUF, M_NOWAIT | M_ZERO);
3668         if (a == NULL) {
3669                 device_printf(dev, "add_hw_filters failed to get memory\n");
3670                 return;
3671         }
3672
3673         /*
3674         ** Scan the filter list, each time we find one
3675         ** we add it to the admin queue array and turn off
3676         ** the add bit.
3677         */
3678         SLIST_FOREACH(f, &vsi->ftl, next) {
3679                 if (f->flags == flags) {
3680                         b = &a[j]; // a pox on fvl long names :)
3681                         bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
3682                         if (f->vlan == IXL_VLAN_ANY) {
3683                                 b->vlan_tag = 0;
3684                                 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
3685                         } else {
3686                                 b->vlan_tag = f->vlan;
3687                                 b->flags = 0;
3688                         }
3689                         b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
3690                         f->flags &= ~IXL_FILTER_ADD;
3691                         j++;
3692                 }
3693                 if (j == cnt)
3694                         break;
3695         }
3696         if (j > 0) {
3697                 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
3698                 if (err)
3699                         device_printf(dev, "aq_add_macvlan err %d, "
3700                             "aq_error %d\n", err, hw->aq.asq_last_status);
3701                 else
3702                         vsi->hw_filters_add += j;
3703         }
3704         free(a, M_DEVBUF);
3705         return;
3706 }
3707
3708 /*
3709 ** This routine takes removals in the vsi filter
3710 ** table and creates an Admin Queue call to delete
3711 ** the filters in the hardware.
3712 */
3713 static void
3714 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
3715 {
3716         struct i40e_aqc_remove_macvlan_element_data *d, *e;
3717         struct ixl_pf           *pf;
3718         struct i40e_hw          *hw;
3719         device_t                dev;
3720         struct ixl_mac_filter   *f, *f_temp;
3721         int                     err, j = 0;
3722
3723         DEBUGOUT("ixl_del_hw_filters: begin\n");
3724
3725         pf = vsi->back;
3726         hw = &pf->hw;
3727         dev = pf->dev;
3728
3729         d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
3730             M_DEVBUF, M_NOWAIT | M_ZERO);
3731         if (d == NULL) {
3732                 printf("del hw filter failed to get memory\n");
3733                 return;
3734         }
3735
3736         SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
3737                 if (f->flags & IXL_FILTER_DEL) {
3738                         e = &d[j]; // a pox on fvl long names :)
3739                         bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
3740                         e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
3741                         e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3742                         /* delete entry from vsi list */
3743                         SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
3744                         free(f, M_DEVBUF);
3745                         j++;
3746                 }
3747                 if (j == cnt)
3748                         break;
3749         }
3750         if (j > 0) {
3751                 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
3752                 /* NOTE: returns ENOENT every time but seems to work fine,
3753                    so we'll ignore that specific error. */
3754                 // TODO: Does this still occur on current firmwares?
3755                 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
3756                         int sc = 0;
3757                         for (int i = 0; i < j; i++)
3758                                 sc += (!d[i].error_code);
3759                         vsi->hw_filters_del += sc;
3760                         device_printf(dev,
3761                             "Failed to remove %d/%d filters, aq error %d\n",
3762                             j - sc, j, hw->aq.asq_last_status);
3763                 } else
3764                         vsi->hw_filters_del += j;
3765         }
3766         free(d, M_DEVBUF);
3767
3768         DEBUGOUT("ixl_del_hw_filters: end\n");
3769         return;
3770 }
3771
3772 static int
3773 ixl_enable_rings(struct ixl_vsi *vsi)
3774 {
3775         struct ixl_pf   *pf = vsi->back;
3776         struct i40e_hw  *hw = &pf->hw;
3777         int             index, error;
3778         u32             reg;
3779
3780         error = 0;
3781         for (int i = 0; i < vsi->num_queues; i++) {
3782                 index = vsi->first_queue + i;
3783                 i40e_pre_tx_queue_cfg(hw, index, TRUE);
3784
3785                 reg = rd32(hw, I40E_QTX_ENA(index));
3786                 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3787                     I40E_QTX_ENA_QENA_STAT_MASK;
3788                 wr32(hw, I40E_QTX_ENA(index), reg);
3789                 /* Verify the enable took */
3790                 for (int j = 0; j < 10; j++) {
3791                         reg = rd32(hw, I40E_QTX_ENA(index));
3792                         if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
3793                                 break;
3794                         i40e_msec_delay(10);
3795                 }
3796                 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
3797                         device_printf(pf->dev, "TX queue %d disabled!\n",
3798                             index);
3799                         error = ETIMEDOUT;
3800                 }
3801
3802                 reg = rd32(hw, I40E_QRX_ENA(index));
3803                 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3804                     I40E_QRX_ENA_QENA_STAT_MASK;
3805                 wr32(hw, I40E_QRX_ENA(index), reg);
3806                 /* Verify the enable took */
3807                 for (int j = 0; j < 10; j++) {
3808                         reg = rd32(hw, I40E_QRX_ENA(index));
3809                         if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3810                                 break;
3811                         i40e_msec_delay(10);
3812                 }
3813                 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
3814                         device_printf(pf->dev, "RX queue %d disabled!\n",
3815                             index);
3816                         error = ETIMEDOUT;
3817                 }
3818         }
3819
3820         return (error);
3821 }
3822
3823 static int
3824 ixl_disable_rings(struct ixl_vsi *vsi)
3825 {
3826         struct ixl_pf   *pf = vsi->back;
3827         struct i40e_hw  *hw = &pf->hw;
3828         int             index, error;
3829         u32             reg;
3830
3831         error = 0;
3832         for (int i = 0; i < vsi->num_queues; i++) {
3833                 index = vsi->first_queue + i;
3834
3835                 i40e_pre_tx_queue_cfg(hw, index, FALSE);
3836                 i40e_usec_delay(500);
3837
3838                 reg = rd32(hw, I40E_QTX_ENA(index));
3839                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3840                 wr32(hw, I40E_QTX_ENA(index), reg);
3841                 /* Verify the disable took */
3842                 for (int j = 0; j < 10; j++) {
3843                         reg = rd32(hw, I40E_QTX_ENA(index));
3844                         if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3845                                 break;
3846                         i40e_msec_delay(10);
3847                 }
3848                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
3849                         device_printf(pf->dev, "TX queue %d still enabled!\n",
3850                             index);
3851                         error = ETIMEDOUT;
3852                 }
3853
3854                 reg = rd32(hw, I40E_QRX_ENA(index));
3855                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3856                 wr32(hw, I40E_QRX_ENA(index), reg);
3857                 /* Verify the disable took */
3858                 for (int j = 0; j < 10; j++) {
3859                         reg = rd32(hw, I40E_QRX_ENA(index));
3860                         if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3861                                 break;
3862                         i40e_msec_delay(10);
3863                 }
3864                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
3865                         device_printf(pf->dev, "RX queue %d still enabled!\n",
3866                             index);
3867                         error = ETIMEDOUT;
3868                 }
3869         }
3870
3871         return (error);
3872 }
3873
3874 /**
3875  * ixl_handle_mdd_event
3876  *
3877  * Called from interrupt handler to identify possibly malicious vfs
3878  * (But also detects events from the PF, as well)
3879  **/
3880 static void ixl_handle_mdd_event(struct ixl_pf *pf)
3881 {
3882         struct i40e_hw *hw = &pf->hw;
3883         device_t dev = pf->dev;
3884         bool mdd_detected = false;
3885         bool pf_mdd_detected = false;
3886         u32 reg;
3887
3888         /* find what triggered the MDD event */
3889         reg = rd32(hw, I40E_GL_MDET_TX);
3890         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
3891                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
3892                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
3893                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
3894                                 I40E_GL_MDET_TX_EVENT_SHIFT;
3895                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
3896                                 I40E_GL_MDET_TX_QUEUE_SHIFT;
3897                 device_printf(dev,
3898                          "Malicious Driver Detection event 0x%02x"
3899                          " on TX queue %d pf number 0x%02x\n",
3900                          event, queue, pf_num);
3901                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
3902                 mdd_detected = true;
3903         }
3904         reg = rd32(hw, I40E_GL_MDET_RX);
3905         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
3906                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
3907                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
3908                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
3909                                 I40E_GL_MDET_RX_EVENT_SHIFT;
3910                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
3911                                 I40E_GL_MDET_RX_QUEUE_SHIFT;
3912                 device_printf(dev,
3913                          "Malicious Driver Detection event 0x%02x"
3914                          " on RX queue %d of function 0x%02x\n",
3915                          event, queue, func);
3916                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
3917                 mdd_detected = true;
3918         }
3919
3920         if (mdd_detected) {
3921                 reg = rd32(hw, I40E_PF_MDET_TX);
3922                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
3923                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
3924                         device_printf(dev,
3925                                  "MDD TX event is for this function 0x%08x",
3926                                  reg);
3927                         pf_mdd_detected = true;
3928                 }
3929                 reg = rd32(hw, I40E_PF_MDET_RX);
3930                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
3931                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
3932                         device_printf(dev,
3933                                  "MDD RX event is for this function 0x%08x",
3934                                  reg);
3935                         pf_mdd_detected = true;
3936                 }
3937         }
3938
3939         /* re-enable mdd interrupt cause */
3940         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
3941         reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3942         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
3943         ixl_flush(hw);
3944 }
3945
3946 static void
3947 ixl_enable_intr(struct ixl_vsi *vsi)
3948 {
3949         struct i40e_hw          *hw = vsi->hw;
3950         struct ixl_queue        *que = vsi->queues;
3951
3952         if (ixl_enable_msix) {
3953                 ixl_enable_adminq(hw);
3954                 for (int i = 0; i < vsi->num_queues; i++, que++)
3955                         ixl_enable_queue(hw, que->me);
3956         } else
3957                 ixl_enable_legacy(hw);
3958 }
3959
3960 static void
3961 ixl_disable_rings_intr(struct ixl_vsi *vsi)
3962 {
3963         struct i40e_hw          *hw = vsi->hw;
3964         struct ixl_queue        *que = vsi->queues;
3965
3966         for (int i = 0; i < vsi->num_queues; i++, que++)
3967                 ixl_disable_queue(hw, que->me);
3968 }
3969
3970 static void
3971 ixl_disable_intr(struct ixl_vsi *vsi)
3972 {
3973         struct i40e_hw          *hw = vsi->hw;
3974
3975         if (ixl_enable_msix)
3976                 ixl_disable_adminq(hw);
3977         else
3978                 ixl_disable_legacy(hw);
3979 }
3980
3981 static void
3982 ixl_enable_adminq(struct i40e_hw *hw)
3983 {
3984         u32             reg;
3985
3986         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
3987             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3988             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3989         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
3990         ixl_flush(hw);
3991         return;
3992 }
3993
3994 static void
3995 ixl_disable_adminq(struct i40e_hw *hw)
3996 {
3997         u32             reg;
3998
3999         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4000         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4001
4002         return;
4003 }
4004
4005 static void
4006 ixl_enable_queue(struct i40e_hw *hw, int id)
4007 {
4008         u32             reg;
4009
4010         reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4011             I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4012             (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4013         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4014 }
4015
4016 static void
4017 ixl_disable_queue(struct i40e_hw *hw, int id)
4018 {
4019         u32             reg;
4020
4021         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4022         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4023
4024         return;
4025 }
4026
4027 static void
4028 ixl_enable_legacy(struct i40e_hw *hw)
4029 {
4030         u32             reg;
4031         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4032             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4033             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4034         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4035 }
4036
4037 static void
4038 ixl_disable_legacy(struct i40e_hw *hw)
4039 {
4040         u32             reg;
4041
4042         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4043         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4044
4045         return;
4046 }
4047
4048 static void
4049 ixl_update_stats_counters(struct ixl_pf *pf)
4050 {
4051         struct i40e_hw  *hw = &pf->hw;
4052         struct ixl_vsi  *vsi = &pf->vsi;
4053         struct ixl_vf   *vf;
4054
4055         struct i40e_hw_port_stats *nsd = &pf->stats;
4056         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4057
4058         /* Update hw stats */
4059         ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4060                            pf->stat_offsets_loaded,
4061                            &osd->crc_errors, &nsd->crc_errors);
4062         ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4063                            pf->stat_offsets_loaded,
4064                            &osd->illegal_bytes, &nsd->illegal_bytes);
4065         ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4066                            I40E_GLPRT_GORCL(hw->port),
4067                            pf->stat_offsets_loaded,
4068                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4069         ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4070                            I40E_GLPRT_GOTCL(hw->port),
4071                            pf->stat_offsets_loaded,
4072                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4073         ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4074                            pf->stat_offsets_loaded,
4075                            &osd->eth.rx_discards,
4076                            &nsd->eth.rx_discards);
4077         ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4078                            I40E_GLPRT_UPRCL(hw->port),
4079                            pf->stat_offsets_loaded,
4080                            &osd->eth.rx_unicast,
4081                            &nsd->eth.rx_unicast);
4082         ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4083                            I40E_GLPRT_UPTCL(hw->port),
4084                            pf->stat_offsets_loaded,
4085                            &osd->eth.tx_unicast,
4086                            &nsd->eth.tx_unicast);
4087         ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4088                            I40E_GLPRT_MPRCL(hw->port),
4089                            pf->stat_offsets_loaded,
4090                            &osd->eth.rx_multicast,
4091                            &nsd->eth.rx_multicast);
4092         ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4093                            I40E_GLPRT_MPTCL(hw->port),
4094                            pf->stat_offsets_loaded,
4095                            &osd->eth.tx_multicast,
4096                            &nsd->eth.tx_multicast);
4097         ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4098                            I40E_GLPRT_BPRCL(hw->port),
4099                            pf->stat_offsets_loaded,
4100                            &osd->eth.rx_broadcast,
4101                            &nsd->eth.rx_broadcast);
4102         ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4103                            I40E_GLPRT_BPTCL(hw->port),
4104                            pf->stat_offsets_loaded,
4105                            &osd->eth.tx_broadcast,
4106                            &nsd->eth.tx_broadcast);
4107
4108         ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4109                            pf->stat_offsets_loaded,
4110                            &osd->tx_dropped_link_down,
4111                            &nsd->tx_dropped_link_down);
4112         ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4113                            pf->stat_offsets_loaded,
4114                            &osd->mac_local_faults,
4115                            &nsd->mac_local_faults);
4116         ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4117                            pf->stat_offsets_loaded,
4118                            &osd->mac_remote_faults,
4119                            &nsd->mac_remote_faults);
4120         ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4121                            pf->stat_offsets_loaded,
4122                            &osd->rx_length_errors,
4123                            &nsd->rx_length_errors);
4124
4125         /* Flow control (LFC) stats */
4126         ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4127                            pf->stat_offsets_loaded,
4128                            &osd->link_xon_rx, &nsd->link_xon_rx);
4129         ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4130                            pf->stat_offsets_loaded,
4131                            &osd->link_xon_tx, &nsd->link_xon_tx);
4132         ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4133                            pf->stat_offsets_loaded,
4134                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
4135         ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4136                            pf->stat_offsets_loaded,
4137                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
4138
4139         /* Packet size stats rx */
4140         ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4141                            I40E_GLPRT_PRC64L(hw->port),
4142                            pf->stat_offsets_loaded,
4143                            &osd->rx_size_64, &nsd->rx_size_64);
4144         ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4145                            I40E_GLPRT_PRC127L(hw->port),
4146                            pf->stat_offsets_loaded,
4147                            &osd->rx_size_127, &nsd->rx_size_127);
4148         ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4149                            I40E_GLPRT_PRC255L(hw->port),
4150                            pf->stat_offsets_loaded,
4151                            &osd->rx_size_255, &nsd->rx_size_255);
4152         ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4153                            I40E_GLPRT_PRC511L(hw->port),
4154                            pf->stat_offsets_loaded,
4155                            &osd->rx_size_511, &nsd->rx_size_511);
4156         ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4157                            I40E_GLPRT_PRC1023L(hw->port),
4158                            pf->stat_offsets_loaded,
4159                            &osd->rx_size_1023, &nsd->rx_size_1023);
4160         ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4161                            I40E_GLPRT_PRC1522L(hw->port),
4162                            pf->stat_offsets_loaded,
4163                            &osd->rx_size_1522, &nsd->rx_size_1522);
4164         ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4165                            I40E_GLPRT_PRC9522L(hw->port),
4166                            pf->stat_offsets_loaded,
4167                            &osd->rx_size_big, &nsd->rx_size_big);
4168
4169         /* Packet size stats tx */
4170         ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4171                            I40E_GLPRT_PTC64L(hw->port),
4172                            pf->stat_offsets_loaded,
4173                            &osd->tx_size_64, &nsd->tx_size_64);
4174         ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4175                            I40E_GLPRT_PTC127L(hw->port),
4176                            pf->stat_offsets_loaded,
4177                            &osd->tx_size_127, &nsd->tx_size_127);
4178         ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4179                            I40E_GLPRT_PTC255L(hw->port),
4180                            pf->stat_offsets_loaded,
4181                            &osd->tx_size_255, &nsd->tx_size_255);
4182         ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4183                            I40E_GLPRT_PTC511L(hw->port),
4184                            pf->stat_offsets_loaded,
4185                            &osd->tx_size_511, &nsd->tx_size_511);
4186         ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4187                            I40E_GLPRT_PTC1023L(hw->port),
4188                            pf->stat_offsets_loaded,
4189                            &osd->tx_size_1023, &nsd->tx_size_1023);
4190         ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4191                            I40E_GLPRT_PTC1522L(hw->port),
4192                            pf->stat_offsets_loaded,
4193                            &osd->tx_size_1522, &nsd->tx_size_1522);
4194         ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4195                            I40E_GLPRT_PTC9522L(hw->port),
4196                            pf->stat_offsets_loaded,
4197                            &osd->tx_size_big, &nsd->tx_size_big);
4198
4199         ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4200                            pf->stat_offsets_loaded,
4201                            &osd->rx_undersize, &nsd->rx_undersize);
4202         ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4203                            pf->stat_offsets_loaded,
4204                            &osd->rx_fragments, &nsd->rx_fragments);
4205         ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4206                            pf->stat_offsets_loaded,
4207                            &osd->rx_oversize, &nsd->rx_oversize);
4208         ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4209                            pf->stat_offsets_loaded,
4210                            &osd->rx_jabber, &nsd->rx_jabber);
4211         pf->stat_offsets_loaded = true;
4212         /* End hw stats */
4213
4214         /* Update vsi stats */
4215         ixl_update_vsi_stats(vsi);
4216
4217         for (int i = 0; i < pf->num_vfs; i++) {
4218                 vf = &pf->vfs[i];
4219                 if (vf->vf_flags & VF_FLAG_ENABLED)
4220                         ixl_update_eth_stats(&pf->vfs[i].vsi);
4221         }
4222 }
4223
4224 /*
4225 ** Tasklet handler for MSIX Adminq interrupts
4226 **  - do outside interrupt since it might sleep
4227 */
4228 static void
4229 ixl_do_adminq(void *context, int pending)
4230 {
4231         struct ixl_pf                   *pf = context;
4232         struct i40e_hw                  *hw = &pf->hw;
4233         struct ixl_vsi                  *vsi = &pf->vsi;
4234         struct i40e_arq_event_info      event;
4235         i40e_status                     ret;
4236         u32                             reg, loop = 0;
4237         u16                             opcode, result;
4238
4239         event.buf_len = IXL_AQ_BUF_SZ;
4240         event.msg_buf = malloc(event.buf_len,
4241             M_DEVBUF, M_NOWAIT | M_ZERO);
4242         if (!event.msg_buf) {
4243                 printf("Unable to allocate adminq memory\n");
4244                 return;
4245         }
4246
4247         IXL_PF_LOCK(pf);
4248         /* clean and process any events */
4249         do {
4250                 ret = i40e_clean_arq_element(hw, &event, &result);
4251                 if (ret)
4252                         break;
4253                 opcode = LE16_TO_CPU(event.desc.opcode);
4254                 switch (opcode) {
4255                 case i40e_aqc_opc_get_link_status:
4256                         ixl_link_event(pf, &event);
4257                         ixl_update_link_status(pf);
4258                         break;
4259                 case i40e_aqc_opc_send_msg_to_pf:
4260 #ifdef PCI_IOV
4261                         ixl_handle_vf_msg(pf, &event);
4262 #endif
4263                         break;
4264                 case i40e_aqc_opc_event_lan_overflow:
4265                         break;
4266                 default:
4267 #ifdef IXL_DEBUG
4268                         printf("AdminQ unknown event %x\n", opcode);
4269 #endif
4270                         break;
4271                 }
4272
4273         } while (result && (loop++ < IXL_ADM_LIMIT));
4274
4275         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4276         reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4277         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4278         free(event.msg_buf, M_DEVBUF);
4279
4280         /*
4281          * If there are still messages to process, reschedule ourselves.
4282          * Otherwise, re-enable our interrupt and go to sleep.
4283          */
4284         if (result > 0)
4285                 taskqueue_enqueue(pf->tq, &pf->adminq);
4286         else
4287                 ixl_enable_intr(vsi);
4288
4289         IXL_PF_UNLOCK(pf);
4290 }
4291
4292 static int
4293 ixl_debug_info(SYSCTL_HANDLER_ARGS)
4294 {
4295         struct ixl_pf   *pf;
4296         int             error, input = 0;
4297
4298         error = sysctl_handle_int(oidp, &input, 0, req);
4299
4300         if (error || !req->newptr)
4301                 return (error);
4302
4303         if (input == 1) {
4304                 pf = (struct ixl_pf *)arg1;
4305                 ixl_print_debug_info(pf);
4306         }
4307
4308         return (error);
4309 }
4310
4311 static void
4312 ixl_print_debug_info(struct ixl_pf *pf)
4313 {
4314         struct i40e_hw          *hw = &pf->hw;
4315         struct ixl_vsi          *vsi = &pf->vsi;
4316         struct ixl_queue        *que = vsi->queues;
4317         struct rx_ring          *rxr = &que->rxr;
4318         struct tx_ring          *txr = &que->txr;
4319         u32                     reg;
4320
4321
4322         printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
4323         printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
4324         printf("RX next check = %x\n", rxr->next_check);
4325         printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
4326         printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
4327         printf("TX desc avail = %x\n", txr->avail);
4328
4329         reg = rd32(hw, I40E_GLV_GORCL(0xc));
4330          printf("RX Bytes = %x\n", reg);
4331         reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
4332          printf("Port RX Bytes = %x\n", reg);
4333         reg = rd32(hw, I40E_GLV_RDPC(0xc));
4334          printf("RX discard = %x\n", reg);
4335         reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
4336          printf("Port RX discard = %x\n", reg);
4337
4338         reg = rd32(hw, I40E_GLV_TEPC(0xc));
4339          printf("TX errors = %x\n", reg);
4340         reg = rd32(hw, I40E_GLV_GOTCL(0xc));
4341          printf("TX Bytes = %x\n", reg);
4342
4343         reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
4344          printf("RX undersize = %x\n", reg);
4345         reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
4346          printf("RX fragments = %x\n", reg);
4347         reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
4348          printf("RX oversize = %x\n", reg);
4349         reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
4350          printf("RX length error = %x\n", reg);
4351         reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
4352          printf("mac remote fault = %x\n", reg);
4353         reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
4354          printf("mac local fault = %x\n", reg);
4355 }
4356
4357 /**
4358  * Update VSI-specific ethernet statistics counters.
4359  **/
4360 void ixl_update_eth_stats(struct ixl_vsi *vsi)
4361 {
4362         struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4363         struct i40e_hw *hw = &pf->hw;
4364         struct i40e_eth_stats *es;
4365         struct i40e_eth_stats *oes;
4366         struct i40e_hw_port_stats *nsd;
4367         u16 stat_idx = vsi->info.stat_counter_idx;
4368
4369         es = &vsi->eth_stats;
4370         oes = &vsi->eth_stats_offsets;
4371         nsd = &pf->stats;
4372
4373         /* Gather up the stats that the hw collects */
4374         ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4375                            vsi->stat_offsets_loaded,
4376                            &oes->tx_errors, &es->tx_errors);
4377         ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4378                            vsi->stat_offsets_loaded,
4379                            &oes->rx_discards, &es->rx_discards);
4380
4381         ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4382                            I40E_GLV_GORCL(stat_idx),
4383                            vsi->stat_offsets_loaded,
4384                            &oes->rx_bytes, &es->rx_bytes);
4385         ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4386                            I40E_GLV_UPRCL(stat_idx),
4387                            vsi->stat_offsets_loaded,
4388                            &oes->rx_unicast, &es->rx_unicast);
4389         ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4390                            I40E_GLV_MPRCL(stat_idx),
4391                            vsi->stat_offsets_loaded,
4392                            &oes->rx_multicast, &es->rx_multicast);
4393         ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4394                            I40E_GLV_BPRCL(stat_idx),
4395                            vsi->stat_offsets_loaded,
4396                            &oes->rx_broadcast, &es->rx_broadcast);
4397
4398         ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4399                            I40E_GLV_GOTCL(stat_idx),
4400                            vsi->stat_offsets_loaded,
4401                            &oes->tx_bytes, &es->tx_bytes);
4402         ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4403                            I40E_GLV_UPTCL(stat_idx),
4404                            vsi->stat_offsets_loaded,
4405                            &oes->tx_unicast, &es->tx_unicast);
4406         ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4407                            I40E_GLV_MPTCL(stat_idx),
4408                            vsi->stat_offsets_loaded,
4409                            &oes->tx_multicast, &es->tx_multicast);
4410         ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4411                            I40E_GLV_BPTCL(stat_idx),
4412                            vsi->stat_offsets_loaded,
4413                            &oes->tx_broadcast, &es->tx_broadcast);
4414         vsi->stat_offsets_loaded = true;
4415 }
4416
4417 static void
4418 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4419 {
4420         struct ixl_pf           *pf;
4421         struct ifnet            *ifp;
4422         struct i40e_eth_stats   *es;
4423         u64                     tx_discards;
4424
4425         struct i40e_hw_port_stats *nsd;
4426
4427         pf = vsi->back;
4428         ifp = vsi->ifp;
4429         es = &vsi->eth_stats;
4430         nsd = &pf->stats;
4431
4432         ixl_update_eth_stats(vsi);
4433
4434         tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4435         for (int i = 0; i < vsi->num_queues; i++)
4436                 tx_discards += vsi->queues[i].txr.br->br_drops;
4437
4438         /* Update ifnet stats */
4439         IXL_SET_IPACKETS(vsi, es->rx_unicast +
4440                            es->rx_multicast +
4441                            es->rx_broadcast);
4442         IXL_SET_OPACKETS(vsi, es->tx_unicast +
4443                            es->tx_multicast +
4444                            es->tx_broadcast);
4445         IXL_SET_IBYTES(vsi, es->rx_bytes);
4446         IXL_SET_OBYTES(vsi, es->tx_bytes);
4447         IXL_SET_IMCASTS(vsi, es->rx_multicast);
4448         IXL_SET_OMCASTS(vsi, es->tx_multicast);
4449
4450         IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
4451             nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
4452             nsd->rx_jabber);
4453         IXL_SET_OERRORS(vsi, es->tx_errors);
4454         IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
4455         IXL_SET_OQDROPS(vsi, tx_discards);
4456         IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
4457         IXL_SET_COLLISIONS(vsi, 0);
4458 }
4459
4460 /**
4461  * Reset all of the stats for the given pf
4462  **/
4463 void ixl_pf_reset_stats(struct ixl_pf *pf)
4464 {
4465         bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
4466         bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
4467         pf->stat_offsets_loaded = false;
4468 }
4469
4470 /**
4471  * Resets all stats of the given vsi
4472  **/
4473 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
4474 {
4475         bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
4476         bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
4477         vsi->stat_offsets_loaded = false;
4478 }
4479
4480 /**
4481  * Read and update a 48 bit stat from the hw
4482  *
4483  * Since the device stats are not reset at PFReset, they likely will not
4484  * be zeroed when the driver starts.  We'll save the first values read
4485  * and use them as offsets to be subtracted from the raw values in order
4486  * to report stats that count from zero.
4487  **/
4488 static void
4489 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
4490         bool offset_loaded, u64 *offset, u64 *stat)
4491 {
4492         u64 new_data;
4493
4494 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
4495         new_data = rd64(hw, loreg);
4496 #else
4497         /*
4498          * Use two rd32's instead of one rd64; FreeBSD versions before
4499          * 10 don't support 8 byte bus reads/writes.
4500          */
4501         new_data = rd32(hw, loreg);
4502         new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4503 #endif
4504
4505         if (!offset_loaded)
4506                 *offset = new_data;
4507         if (new_data >= *offset)
4508                 *stat = new_data - *offset;
4509         else
4510                 *stat = (new_data + ((u64)1 << 48)) - *offset;
4511         *stat &= 0xFFFFFFFFFFFFULL;
4512 }
4513
4514 /**
4515  * Read and update a 32 bit stat from the hw
4516  **/
4517 static void
4518 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
4519         bool offset_loaded, u64 *offset, u64 *stat)
4520 {
4521         u32 new_data;
4522
4523         new_data = rd32(hw, reg);
4524         if (!offset_loaded)
4525                 *offset = new_data;
4526         if (new_data >= *offset)
4527                 *stat = (u32)(new_data - *offset);
4528         else
4529                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
4530 }
4531
4532 /*
4533 ** Set flow control using sysctl:
4534 **      0 - off
4535 **      1 - rx pause
4536 **      2 - tx pause
4537 **      3 - full
4538 */
4539 static int
4540 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
4541 {
4542         /*
4543          * TODO: ensure flow control is disabled if
4544          * priority flow control is enabled
4545          *
4546          * TODO: ensure tx CRC by hardware should be enabled
4547          * if tx flow control is enabled.
4548          */
4549         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4550         struct i40e_hw *hw = &pf->hw;
4551         device_t dev = pf->dev;
4552         int error = 0;
4553         enum i40e_status_code aq_error = 0;
4554         u8 fc_aq_err = 0;
4555
4556         /* Get request */
4557         error = sysctl_handle_int(oidp, &pf->fc, 0, req);
4558         if ((error) || (req->newptr == NULL))
4559                 return (error);
4560         if (pf->fc < 0 || pf->fc > 3) {
4561                 device_printf(dev,
4562                     "Invalid fc mode; valid modes are 0 through 3\n");
4563                 return (EINVAL);
4564         }
4565
4566         /*
4567         ** Changing flow control mode currently does not work on
4568         ** 40GBASE-CR4 PHYs
4569         */
4570         if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4
4571             || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) {
4572                 device_printf(dev, "Changing flow control mode unsupported"
4573                     " on 40GBase-CR4 media.\n");
4574                 return (ENODEV);
4575         }
4576
4577         /* Set fc ability for port */
4578         hw->fc.requested_mode = pf->fc;
4579         aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
4580         if (aq_error) {
4581                 device_printf(dev,
4582                     "%s: Error setting new fc mode %d; fc_err %#x\n",
4583                     __func__, aq_error, fc_aq_err);
4584                 return (EAGAIN);
4585         }
4586
4587         return (0);
4588 }
4589
4590 static int
4591 ixl_current_speed(SYSCTL_HANDLER_ARGS)
4592 {
4593         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4594         struct i40e_hw *hw = &pf->hw;
4595         int error = 0, index = 0;
4596
4597         char *speeds[] = {
4598                 "Unknown",
4599                 "100M",
4600                 "1G",
4601                 "10G",
4602                 "40G",
4603                 "20G"
4604         };
4605
4606         ixl_update_link_status(pf);
4607
4608         switch (hw->phy.link_info.link_speed) {
4609         case I40E_LINK_SPEED_100MB:
4610                 index = 1;
4611                 break;
4612         case I40E_LINK_SPEED_1GB:
4613                 index = 2;
4614                 break;
4615         case I40E_LINK_SPEED_10GB:
4616                 index = 3;
4617                 break;
4618         case I40E_LINK_SPEED_40GB:
4619                 index = 4;
4620                 break;
4621         case I40E_LINK_SPEED_20GB:
4622                 index = 5;
4623                 break;
4624         case I40E_LINK_SPEED_UNKNOWN:
4625         default:
4626                 index = 0;
4627                 break;
4628         }
4629
4630         error = sysctl_handle_string(oidp, speeds[index],
4631             strlen(speeds[index]), req);
4632         return (error);
4633 }
4634
4635 static int
4636 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
4637 {
4638         struct i40e_hw *hw = &pf->hw;
4639         device_t dev = pf->dev;
4640         struct i40e_aq_get_phy_abilities_resp abilities;
4641         struct i40e_aq_set_phy_config config;
4642         enum i40e_status_code aq_error = 0;
4643
4644         /* Get current capability information */
4645         aq_error = i40e_aq_get_phy_capabilities(hw,
4646             FALSE, FALSE, &abilities, NULL);
4647         if (aq_error) {
4648                 device_printf(dev,
4649                     "%s: Error getting phy capabilities %d,"
4650                     " aq error: %d\n", __func__, aq_error,
4651                     hw->aq.asq_last_status);
4652                 return (EAGAIN);
4653         }
4654
4655         /* Prepare new config */
4656         bzero(&config, sizeof(config));
4657         config.phy_type = abilities.phy_type;
4658         config.abilities = abilities.abilities
4659             | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
4660         config.eee_capability = abilities.eee_capability;
4661         config.eeer = abilities.eeer_val;
4662         config.low_power_ctrl = abilities.d3_lpan;
4663         /* Translate into aq cmd link_speed */
4664         if (speeds & 0x8)
4665                 config.link_speed |= I40E_LINK_SPEED_20GB;
4666         if (speeds & 0x4)
4667                 config.link_speed |= I40E_LINK_SPEED_10GB;
4668         if (speeds & 0x2)
4669                 config.link_speed |= I40E_LINK_SPEED_1GB;
4670         if (speeds & 0x1)
4671                 config.link_speed |= I40E_LINK_SPEED_100MB;
4672
4673         /* Do aq command & restart link */
4674         aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
4675         if (aq_error) {
4676                 device_printf(dev,
4677                     "%s: Error setting new phy config %d,"
4678                     " aq error: %d\n", __func__, aq_error,
4679                     hw->aq.asq_last_status);
4680                 return (EAGAIN);
4681         }
4682
4683         /*
4684         ** This seems a bit heavy handed, but we
4685         ** need to get a reinit on some devices
4686         */
4687         IXL_PF_LOCK(pf);
4688         ixl_stop(pf);
4689         ixl_init_locked(pf);
4690         IXL_PF_UNLOCK(pf);
4691
4692         return (0);
4693 }
4694
4695 /*
4696 ** Control link advertise speed:
4697 **      Flags:
4698 **      0x1 - advertise 100 Mb
4699 **      0x2 - advertise 1G
4700 **      0x4 - advertise 10G
4701 **      0x8 - advertise 20G
4702 **
4703 ** Does not work on 40G devices.
4704 */
4705 static int
4706 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
4707 {
4708         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4709         struct i40e_hw *hw = &pf->hw;
4710         device_t dev = pf->dev;
4711         int requested_ls = 0;
4712         int error = 0;
4713
4714         /*
4715         ** FW doesn't support changing advertised speed
4716         ** for 40G devices; speed is always 40G.
4717         */
4718         if (i40e_is_40G_device(hw->device_id))
4719                 return (ENODEV);
4720
4721         /* Read in new mode */
4722         requested_ls = pf->advertised_speed;
4723         error = sysctl_handle_int(oidp, &requested_ls, 0, req);
4724         if ((error) || (req->newptr == NULL))
4725                 return (error);
4726         /* Check for sane value */
4727         if (requested_ls < 0x1 || requested_ls > 0xE) {
4728                 device_printf(dev, "Invalid advertised speed; "
4729                     "valid modes are 0x1 through 0xE\n");
4730                 return (EINVAL);
4731         }
4732         /* Then check for validity based on adapter type */
4733         switch (hw->device_id) {
4734         case I40E_DEV_ID_10G_BASE_T:
4735                 if (requested_ls & 0x8) {
4736                         device_printf(dev,
4737                             "20Gbs speed not supported on this device.\n");
4738                         return (EINVAL);
4739                 }
4740                 break;
4741         case I40E_DEV_ID_20G_KR2:
4742                 if (requested_ls & 0x1) {
4743                         device_printf(dev,
4744                             "100Mbs speed not supported on this device.\n");
4745                         return (EINVAL);
4746                 }
4747                 break;
4748         default:
4749                 if (requested_ls & ~0x6) {
4750                         device_printf(dev,
4751                             "Only 1/10Gbs speeds are supported on this device.\n");
4752                         return (EINVAL);
4753                 }
4754                 break;
4755         }
4756
4757         /* Exit if no change */
4758         if (pf->advertised_speed == requested_ls)
4759                 return (0);
4760
4761         error = ixl_set_advertised_speeds(pf, requested_ls);
4762         if (error)
4763                 return (error);
4764
4765         pf->advertised_speed = requested_ls;
4766         ixl_update_link_status(pf);
4767         return (0);
4768 }
4769
4770 /*
4771 ** Get the width and transaction speed of
4772 ** the bus this adapter is plugged into.
4773 */
4774 static u16
4775 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
4776 {
4777         u16                     link;
4778         u32                     offset;
4779
4780
4781         /* Get the PCI Express Capabilities offset */
4782         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4783
4784         /* ...and read the Link Status Register */
4785         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4786
4787         switch (link & I40E_PCI_LINK_WIDTH) {
4788         case I40E_PCI_LINK_WIDTH_1:
4789                 hw->bus.width = i40e_bus_width_pcie_x1;
4790                 break;
4791         case I40E_PCI_LINK_WIDTH_2:
4792                 hw->bus.width = i40e_bus_width_pcie_x2;
4793                 break;
4794         case I40E_PCI_LINK_WIDTH_4:
4795                 hw->bus.width = i40e_bus_width_pcie_x4;
4796                 break;
4797         case I40E_PCI_LINK_WIDTH_8:
4798                 hw->bus.width = i40e_bus_width_pcie_x8;
4799                 break;
4800         default:
4801                 hw->bus.width = i40e_bus_width_unknown;
4802                 break;
4803         }
4804
4805         switch (link & I40E_PCI_LINK_SPEED) {
4806         case I40E_PCI_LINK_SPEED_2500:
4807                 hw->bus.speed = i40e_bus_speed_2500;
4808                 break;
4809         case I40E_PCI_LINK_SPEED_5000:
4810                 hw->bus.speed = i40e_bus_speed_5000;
4811                 break;
4812         case I40E_PCI_LINK_SPEED_8000:
4813                 hw->bus.speed = i40e_bus_speed_8000;
4814                 break;
4815         default:
4816                 hw->bus.speed = i40e_bus_speed_unknown;
4817                 break;
4818         }
4819
4820
4821         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4822             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
4823             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
4824             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4825             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
4826             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
4827             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
4828             ("Unknown"));
4829
4830         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
4831             (hw->bus.speed < i40e_bus_speed_8000)) {
4832                 device_printf(dev, "PCI-Express bandwidth available"
4833                     " for this device\n     may be insufficient for"
4834                     " optimal performance.\n");
4835                 device_printf(dev, "For expected performance a x8 "
4836                     "PCIE Gen3 slot is required.\n");
4837         }
4838
4839         return (link);
4840 }
4841
4842 static int
4843 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
4844 {
4845         struct ixl_pf   *pf = (struct ixl_pf *)arg1;
4846         struct i40e_hw  *hw = &pf->hw;
4847         char            buf[32];
4848
4849         snprintf(buf, sizeof(buf),
4850             "f%d.%d a%d.%d n%02x.%02x e%08x",
4851             hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
4852             hw->aq.api_maj_ver, hw->aq.api_min_ver,
4853             (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >>
4854             IXL_NVM_VERSION_HI_SHIFT,
4855             (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >>
4856             IXL_NVM_VERSION_LO_SHIFT,
4857             hw->nvm.eetrack);
4858         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4859 }
4860
4861
4862 #ifdef IXL_DEBUG_SYSCTL
4863 static int
4864 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
4865 {
4866         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4867         struct i40e_hw *hw = &pf->hw;
4868         struct i40e_link_status link_status;
4869         char buf[512];
4870
4871         enum i40e_status_code aq_error = 0;
4872
4873         aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
4874         if (aq_error) {
4875                 printf("i40e_aq_get_link_info() error %d\n", aq_error);
4876                 return (EPERM);
4877         }
4878
4879         sprintf(buf, "\n"
4880             "PHY Type : %#04x\n"
4881             "Speed    : %#04x\n"
4882             "Link info: %#04x\n"
4883             "AN info  : %#04x\n"
4884             "Ext info : %#04x",
4885             link_status.phy_type, link_status.link_speed,
4886             link_status.link_info, link_status.an_info,
4887             link_status.ext_info);
4888
4889         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4890 }
4891
4892 static int
4893 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
4894 {
4895         struct ixl_pf           *pf = (struct ixl_pf *)arg1;
4896         struct i40e_hw          *hw = &pf->hw;
4897         char                    buf[512];
4898         enum i40e_status_code   aq_error = 0;
4899
4900         struct i40e_aq_get_phy_abilities_resp abilities;
4901
4902         aq_error = i40e_aq_get_phy_capabilities(hw,
4903             TRUE, FALSE, &abilities, NULL);
4904         if (aq_error) {
4905                 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
4906                 return (EPERM);
4907         }
4908
4909         sprintf(buf, "\n"
4910             "PHY Type : %#010x\n"
4911             "Speed    : %#04x\n"
4912             "Abilities: %#04x\n"
4913             "EEE cap  : %#06x\n"
4914             "EEER reg : %#010x\n"
4915             "D3 Lpan  : %#04x",
4916             abilities.phy_type, abilities.link_speed,
4917             abilities.abilities, abilities.eee_capability,
4918             abilities.eeer_val, abilities.d3_lpan);
4919
4920         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4921 }
4922
4923 static int
4924 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
4925 {
4926         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4927         struct ixl_vsi *vsi = &pf->vsi;
4928         struct ixl_mac_filter *f;
4929         char *buf, *buf_i;
4930
4931         int error = 0;
4932         int ftl_len = 0;
4933         int ftl_counter = 0;
4934         int buf_len = 0;
4935         int entry_len = 42;
4936
4937         SLIST_FOREACH(f, &vsi->ftl, next) {
4938                 ftl_len++;
4939         }
4940
4941         if (ftl_len < 1) {
4942                 sysctl_handle_string(oidp, "(none)", 6, req);
4943                 return (0);
4944         }
4945
4946         buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
4947         buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
4948
4949         sprintf(buf_i++, "\n");
4950         SLIST_FOREACH(f, &vsi->ftl, next) {
4951                 sprintf(buf_i,
4952                     MAC_FORMAT ", vlan %4d, flags %#06x",
4953                     MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
4954                 buf_i += entry_len;
4955                 /* don't print '\n' for last entry */
4956                 if (++ftl_counter != ftl_len) {
4957                         sprintf(buf_i, "\n");
4958                         buf_i++;
4959                 }
4960         }
4961
4962         error = sysctl_handle_string(oidp, buf, strlen(buf), req);
4963         if (error)
4964                 printf("sysctl error: %d\n", error);
4965         free(buf, M_DEVBUF);
4966         return error;
4967 }
4968
4969 #define IXL_SW_RES_SIZE 0x14
4970 static int
4971 ixl_res_alloc_cmp(const void *a, const void *b)
4972 {
4973         const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
4974         one = (struct i40e_aqc_switch_resource_alloc_element_resp *)a;
4975         two = (struct i40e_aqc_switch_resource_alloc_element_resp *)b;
4976
4977         return ((int)one->resource_type - (int)two->resource_type);
4978 }
4979
4980 static int
4981 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
4982 {
4983         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4984         struct i40e_hw *hw = &pf->hw;
4985         device_t dev = pf->dev;
4986         struct sbuf *buf;
4987         int error = 0;
4988
4989         u8 num_entries;
4990         struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
4991
4992         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4993         if (!buf) {
4994                 device_printf(dev, "Could not allocate sbuf for output.\n");
4995                 return (ENOMEM);
4996         }
4997
4998         bzero(resp, sizeof(resp));
4999         error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5000                                 resp,
5001                                 IXL_SW_RES_SIZE,
5002                                 NULL);
5003         if (error) {
5004                 device_printf(dev,
5005                     "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5006                     __func__, error, hw->aq.asq_last_status);
5007                 sbuf_delete(buf);
5008                 return error;
5009         }
5010
5011         /* Sort entries by type for display */
5012         qsort(resp, num_entries,
5013             sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5014             &ixl_res_alloc_cmp);
5015
5016         sbuf_cat(buf, "\n");
5017         sbuf_printf(buf, "# of entries: %d\n", num_entries);
5018         sbuf_printf(buf,
5019             "Type | Guaranteed | Total | Used   | Un-allocated\n"
5020             "     | (this)     | (all) | (this) | (all)       \n");
5021         for (int i = 0; i < num_entries; i++) {
5022                 sbuf_printf(buf,
5023                     "%#4x | %10d   %5d   %6d   %12d",
5024                     resp[i].resource_type,
5025                     resp[i].guaranteed,
5026                     resp[i].total,
5027                     resp[i].used,
5028                     resp[i].total_unalloced);
5029                 if (i < num_entries - 1)
5030                         sbuf_cat(buf, "\n");
5031         }
5032
5033         error = sbuf_finish(buf);
5034         if (error) {
5035                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5036         }
5037         sbuf_delete(buf);
5038
5039         return (error);
5040 }
5041
5042 /*
5043 ** Caller must init and delete sbuf; this function will clear and
5044 ** finish it for caller.
5045 */
5046 static char *
5047 ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink)
5048 {
5049         sbuf_clear(s);
5050
5051         if (seid == 0 && uplink)
5052                 sbuf_cat(s, "Network");
5053         else if (seid == 0)
5054                 sbuf_cat(s, "Host");
5055         else if (seid == 1)
5056                 sbuf_cat(s, "EMP");
5057         else if (seid <= 5)
5058                 sbuf_printf(s, "MAC %d", seid - 2);
5059         else if (seid <= 15)
5060                 sbuf_cat(s, "Reserved");
5061         else if (seid <= 31)
5062                 sbuf_printf(s, "PF %d", seid - 16);
5063         else if (seid <= 159)
5064                 sbuf_printf(s, "VF %d", seid - 32);
5065         else if (seid <= 287)
5066                 sbuf_cat(s, "Reserved");
5067         else if (seid <= 511)
5068                 sbuf_cat(s, "Other"); // for other structures
5069         else if (seid <= 895)
5070                 sbuf_printf(s, "VSI %d", seid - 512);
5071         else if (seid <= 1023)
5072                 sbuf_printf(s, "Reserved");
5073         else
5074                 sbuf_cat(s, "Invalid");
5075
5076         sbuf_finish(s);
5077         return sbuf_data(s);
5078 }
5079
5080 static int
5081 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5082 {
5083         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5084         struct i40e_hw *hw = &pf->hw;
5085         device_t dev = pf->dev;
5086         struct sbuf *buf;
5087         struct sbuf *nmbuf;
5088         int error = 0;
5089         u8 aq_buf[I40E_AQ_LARGE_BUF];
5090
5091         u16 next = 0;
5092         struct i40e_aqc_get_switch_config_resp *sw_config;
5093         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5094
5095         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5096         if (!buf) {
5097                 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5098                 return (ENOMEM);
5099         }
5100
5101         error = i40e_aq_get_switch_config(hw, sw_config,
5102             sizeof(aq_buf), &next, NULL);
5103         if (error) {
5104                 device_printf(dev,
5105                     "%s: aq_get_switch_config() error %d, aq error %d\n",
5106                     __func__, error, hw->aq.asq_last_status);
5107                 sbuf_delete(buf);
5108                 return error;
5109         }
5110
5111         nmbuf = sbuf_new_auto();
5112         if (!nmbuf) {
5113                 device_printf(dev, "Could not allocate sbuf for name output.\n");
5114                 sbuf_delete(buf);
5115                 return (ENOMEM);
5116         }
5117
5118         sbuf_cat(buf, "\n");
5119         // Assuming <= 255 elements in switch
5120         sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported);
5121         /* Exclude:
5122         ** Revision -- all elements are revision 1 for now
5123         */
5124         sbuf_printf(buf,
5125             "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5126             "                |          |          | (uplink)\n");
5127         for (int i = 0; i < sw_config->header.num_reported; i++) {
5128                 // "%4d (%8s) | %8s   %8s   %#8x",
5129                 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5130                 sbuf_cat(buf, " ");
5131                 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5132                     sw_config->element[i].seid, false));
5133                 sbuf_cat(buf, " | ");
5134                 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5135                     sw_config->element[i].uplink_seid, true));
5136                 sbuf_cat(buf, "   ");
5137                 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5138                     sw_config->element[i].downlink_seid, false));
5139                 sbuf_cat(buf, "   ");
5140                 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5141                 if (i < sw_config->header.num_reported - 1)
5142                         sbuf_cat(buf, "\n");
5143         }
5144         sbuf_delete(nmbuf);
5145
5146         error = sbuf_finish(buf);
5147         if (error) {
5148                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5149         }
5150         sbuf_delete(buf);
5151
5152         return (error);
5153 }
5154 #endif /* IXL_DEBUG_SYSCTL */
5155
5156
5157 #ifdef PCI_IOV
5158 static int
5159 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5160 {
5161         struct i40e_hw *hw;
5162         struct ixl_vsi *vsi;
5163         struct i40e_vsi_context vsi_ctx;
5164         int i;
5165         uint16_t first_queue;
5166         enum i40e_status_code code;
5167
5168         hw = &pf->hw;
5169         vsi = &pf->vsi;
5170
5171         vsi_ctx.pf_num = hw->pf_id;
5172         vsi_ctx.uplink_seid = pf->veb_seid;
5173         vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
5174         vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5175         vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
5176
5177         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
5178
5179         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5180         vsi_ctx.info.switch_id = htole16(0);
5181
5182         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
5183         vsi_ctx.info.sec_flags = 0;
5184         if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
5185                 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
5186
5187         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
5188         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
5189             I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5190
5191         vsi_ctx.info.valid_sections |=
5192             htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
5193         vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
5194         first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
5195         for (i = 0; i < IXLV_MAX_QUEUES; i++)
5196                 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
5197         for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
5198                 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
5199
5200         vsi_ctx.info.tc_mapping[0] = htole16(
5201             (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5202             (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
5203
5204         code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
5205         if (code != I40E_SUCCESS)
5206                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5207         vf->vsi.seid = vsi_ctx.seid;
5208         vf->vsi.vsi_num = vsi_ctx.vsi_number;
5209         vf->vsi.first_queue = first_queue;
5210         vf->vsi.num_queues = IXLV_MAX_QUEUES;
5211
5212         code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
5213         if (code != I40E_SUCCESS)
5214                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5215
5216         code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
5217         if (code != I40E_SUCCESS) {
5218                 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
5219                     ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5220                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5221         }
5222
5223         memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
5224         return (0);
5225 }
5226
5227 static int
5228 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5229 {
5230         struct i40e_hw *hw;
5231         int error;
5232
5233         hw = &pf->hw;
5234
5235         error = ixl_vf_alloc_vsi(pf, vf);
5236         if (error != 0)
5237                 return (error);
5238
5239         vf->vsi.hw_filters_add = 0;
5240         vf->vsi.hw_filters_del = 0;
5241         ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
5242         ixl_reconfigure_filters(&vf->vsi);
5243
5244         return (0);
5245 }
5246
5247 static void
5248 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
5249     uint32_t val)
5250 {
5251         uint32_t qtable;
5252         int index, shift;
5253
5254         /*
5255          * Two queues are mapped in a single register, so we have to do some
5256          * gymnastics to convert the queue number into a register index and
5257          * shift.
5258          */
5259         index = qnum / 2;
5260         shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
5261
5262         qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
5263         qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
5264         qtable |= val << shift;
5265         wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
5266 }
5267
5268 static void
5269 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
5270 {
5271         struct i40e_hw *hw;
5272         uint32_t qtable;
5273         int i;
5274
5275         hw = &pf->hw;
5276
5277         /*
5278          * Contiguous mappings aren't actually supported by the hardware,
5279          * so we have to use non-contiguous mappings.
5280          */
5281         wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
5282              I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
5283
5284         wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
5285             I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
5286
5287         for (i = 0; i < vf->vsi.num_queues; i++) {
5288                 qtable = (vf->vsi.first_queue + i) <<
5289                     I40E_VPLAN_QTABLE_QINDEX_SHIFT;
5290
5291                 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
5292         }
5293
5294         /* Map queues allocated to VF to its VSI. */
5295         for (i = 0; i < vf->vsi.num_queues; i++)
5296                 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
5297
5298         /* Set rest of VSI queues as unused. */
5299         for (; i < IXL_MAX_VSI_QUEUES; i++)
5300                 ixl_vf_map_vsi_queue(hw, vf, i,
5301                     I40E_VSILAN_QTABLE_QINDEX_0_MASK);
5302
5303         ixl_flush(hw);
5304 }
5305
5306 static void
5307 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
5308 {
5309         struct i40e_hw *hw;
5310
5311         hw = &pf->hw;
5312
5313         if (vsi->seid == 0)
5314                 return;
5315
5316         i40e_aq_delete_element(hw, vsi->seid, NULL);
5317 }
5318
5319 static void
5320 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
5321 {
5322
5323         wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
5324         ixl_flush(hw);
5325 }
5326
5327 static void
5328 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
5329 {
5330
5331         wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
5332             I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
5333         ixl_flush(hw);
5334 }
5335
5336 static void
5337 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
5338 {
5339         struct i40e_hw *hw;
5340         uint32_t vfint_reg, vpint_reg;
5341         int i;
5342
5343         hw = &pf->hw;
5344
5345         ixl_vf_vsi_release(pf, &vf->vsi);
5346
5347         /* Index 0 has a special register. */
5348         ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
5349
5350         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5351                 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
5352                 ixl_vf_disable_queue_intr(hw, vfint_reg);
5353         }
5354
5355         /* Index 0 has a special register. */
5356         ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
5357
5358         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5359                 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
5360                 ixl_vf_unregister_intr(hw, vpint_reg);
5361         }
5362
5363         vf->vsi.num_queues = 0;
5364 }
5365
5366 static int
5367 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
5368 {
5369         struct i40e_hw *hw;
5370         int i;
5371         uint16_t global_vf_num;
5372         uint32_t ciad;
5373
5374         hw = &pf->hw;
5375         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5376
5377         wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
5378              (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
5379         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5380                 ciad = rd32(hw, I40E_PF_PCI_CIAD);
5381                 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
5382                         return (0);
5383                 DELAY(1);
5384         }
5385
5386         return (ETIMEDOUT);
5387 }
5388
5389 static void
5390 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5391 {
5392         struct i40e_hw *hw;
5393         uint32_t vfrtrig;
5394
5395         hw = &pf->hw;
5396
5397         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5398         vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5399         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5400         ixl_flush(hw);
5401
5402         ixl_reinit_vf(pf, vf);
5403 }
5404
5405 static void
5406 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5407 {
5408         struct i40e_hw *hw;
5409         uint32_t vfrstat, vfrtrig;
5410         int i, error;
5411
5412         hw = &pf->hw;
5413
5414         error = ixl_flush_pcie(pf, vf);
5415         if (error != 0)
5416                 device_printf(pf->dev,
5417                     "Timed out waiting for PCIe activity to stop on VF-%d\n",
5418                     vf->vf_num);
5419
5420         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5421                 DELAY(10);
5422
5423                 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
5424                 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
5425                         break;
5426         }
5427
5428         if (i == IXL_VF_RESET_TIMEOUT)
5429                 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
5430
5431         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
5432
5433         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5434         vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5435         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5436
5437         if (vf->vsi.seid != 0)
5438                 ixl_disable_rings(&vf->vsi);
5439
5440         ixl_vf_release_resources(pf, vf);
5441         ixl_vf_setup_vsi(pf, vf);
5442         ixl_vf_map_queues(pf, vf);
5443
5444         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
5445         ixl_flush(hw);
5446 }
5447
5448 static const char *
5449 ixl_vc_opcode_str(uint16_t op)
5450 {
5451
5452         switch (op) {
5453         case I40E_VIRTCHNL_OP_VERSION:
5454                 return ("VERSION");
5455         case I40E_VIRTCHNL_OP_RESET_VF:
5456                 return ("RESET_VF");
5457         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
5458                 return ("GET_VF_RESOURCES");
5459         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
5460                 return ("CONFIG_TX_QUEUE");
5461         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
5462                 return ("CONFIG_RX_QUEUE");
5463         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
5464                 return ("CONFIG_VSI_QUEUES");
5465         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
5466                 return ("CONFIG_IRQ_MAP");
5467         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
5468                 return ("ENABLE_QUEUES");
5469         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
5470                 return ("DISABLE_QUEUES");
5471         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
5472                 return ("ADD_ETHER_ADDRESS");
5473         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
5474                 return ("DEL_ETHER_ADDRESS");
5475         case I40E_VIRTCHNL_OP_ADD_VLAN:
5476                 return ("ADD_VLAN");
5477         case I40E_VIRTCHNL_OP_DEL_VLAN:
5478                 return ("DEL_VLAN");
5479         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
5480                 return ("CONFIG_PROMISCUOUS_MODE");
5481         case I40E_VIRTCHNL_OP_GET_STATS:
5482                 return ("GET_STATS");
5483         case I40E_VIRTCHNL_OP_FCOE:
5484                 return ("FCOE");
5485         case I40E_VIRTCHNL_OP_EVENT:
5486                 return ("EVENT");
5487         default:
5488                 return ("UNKNOWN");
5489         }
5490 }
5491
5492 static int
5493 ixl_vc_opcode_level(uint16_t opcode)
5494 {
5495
5496         switch (opcode) {
5497         case I40E_VIRTCHNL_OP_GET_STATS:
5498                 return (10);
5499         default:
5500                 return (5);
5501         }
5502 }
5503
5504 static void
5505 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5506     enum i40e_status_code status, void *msg, uint16_t len)
5507 {
5508         struct i40e_hw *hw;
5509         int global_vf_id;
5510
5511         hw = &pf->hw;
5512         global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
5513
5514         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
5515             "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
5516             ixl_vc_opcode_str(op), op, status, vf->vf_num);
5517
5518         i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
5519 }
5520
5521 static void
5522 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
5523 {
5524
5525         ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
5526 }
5527
5528 static void
5529 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5530     enum i40e_status_code status, const char *file, int line)
5531 {
5532
5533         I40E_VC_DEBUG(pf, 1,
5534             "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
5535             ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
5536         ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
5537 }
5538
5539 static void
5540 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5541     uint16_t msg_size)
5542 {
5543         struct i40e_virtchnl_version_info reply;
5544
5545         if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
5546                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
5547                     I40E_ERR_PARAM);
5548                 return;
5549         }
5550
5551         reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
5552         reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
5553         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
5554             sizeof(reply));
5555 }
5556
5557 static void
5558 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5559     uint16_t msg_size)
5560 {
5561
5562         if (msg_size != 0) {
5563                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
5564                     I40E_ERR_PARAM);
5565                 return;
5566         }
5567
5568         ixl_reset_vf(pf, vf);
5569
5570         /* No response to a reset message. */
5571 }
5572
5573 static void
5574 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5575     uint16_t msg_size)
5576 {
5577         struct i40e_virtchnl_vf_resource reply;
5578
5579         if (msg_size != 0) {
5580                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5581                     I40E_ERR_PARAM);
5582                 return;
5583         }
5584
5585         bzero(&reply, sizeof(reply));
5586
5587         reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2;
5588
5589         reply.num_vsis = 1;
5590         reply.num_queue_pairs = vf->vsi.num_queues;
5591         reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
5592         reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
5593         reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
5594         reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
5595         memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
5596
5597         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5598             I40E_SUCCESS, &reply, sizeof(reply));
5599 }
5600
5601 static int
5602 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5603     struct i40e_virtchnl_txq_info *info)
5604 {
5605         struct i40e_hw *hw;
5606         struct i40e_hmc_obj_txq txq;
5607         uint16_t global_queue_num, global_vf_num;
5608         enum i40e_status_code status;
5609         uint32_t qtx_ctl;
5610
5611         hw = &pf->hw;
5612         global_queue_num = vf->vsi.first_queue + info->queue_id;
5613         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5614         bzero(&txq, sizeof(txq));
5615
5616         status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
5617         if (status != I40E_SUCCESS)
5618                 return (EINVAL);
5619
5620         txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
5621
5622         txq.head_wb_ena = info->headwb_enabled;
5623         txq.head_wb_addr = info->dma_headwb_addr;
5624         txq.qlen = info->ring_len;
5625         txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
5626         txq.rdylist_act = 0;
5627
5628         status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
5629         if (status != I40E_SUCCESS)
5630                 return (EINVAL);
5631
5632         qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
5633             (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
5634             (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
5635         wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
5636         ixl_flush(hw);
5637
5638         return (0);
5639 }
5640
5641 static int
5642 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5643     struct i40e_virtchnl_rxq_info *info)
5644 {
5645         struct i40e_hw *hw;
5646         struct i40e_hmc_obj_rxq rxq;
5647         uint16_t global_queue_num;
5648         enum i40e_status_code status;
5649
5650         hw = &pf->hw;
5651         global_queue_num = vf->vsi.first_queue + info->queue_id;
5652         bzero(&rxq, sizeof(rxq));
5653
5654         if (info->databuffer_size > IXL_VF_MAX_BUFFER)
5655                 return (EINVAL);
5656
5657         if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
5658             info->max_pkt_size < ETHER_MIN_LEN)
5659                 return (EINVAL);
5660
5661         if (info->splithdr_enabled) {
5662                 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
5663                         return (EINVAL);
5664
5665                 rxq.hsplit_0 = info->rx_split_pos &
5666                     (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
5667                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
5668                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
5669                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
5670                 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
5671
5672                 rxq.dtype = 2;
5673         }
5674
5675         status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
5676         if (status != I40E_SUCCESS)
5677                 return (EINVAL);
5678
5679         rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
5680         rxq.qlen = info->ring_len;
5681
5682         rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
5683
5684         rxq.dsize = 1;
5685         rxq.crcstrip = 1;
5686         rxq.l2tsel = 1;
5687
5688         rxq.rxmax = info->max_pkt_size;
5689         rxq.tphrdesc_ena = 1;
5690         rxq.tphwdesc_ena = 1;
5691         rxq.tphdata_ena = 1;
5692         rxq.tphhead_ena = 1;
5693         rxq.lrxqthresh = 2;
5694         rxq.prefena = 1;
5695
5696         status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
5697         if (status != I40E_SUCCESS)
5698                 return (EINVAL);
5699
5700         return (0);
5701 }
5702
5703 static void
5704 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5705     uint16_t msg_size)
5706 {
5707         struct i40e_virtchnl_vsi_queue_config_info *info;
5708         struct i40e_virtchnl_queue_pair_info *pair;
5709         int i;
5710
5711         if (msg_size < sizeof(*info)) {
5712                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5713                     I40E_ERR_PARAM);
5714                 return;
5715         }
5716
5717         info = msg;
5718         if (info->num_queue_pairs == 0) {
5719                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5720                     I40E_ERR_PARAM);
5721                 return;
5722         }
5723
5724         if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
5725                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5726                     I40E_ERR_PARAM);
5727                 return;
5728         }
5729
5730         if (info->vsi_id != vf->vsi.vsi_num) {
5731                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5732                     I40E_ERR_PARAM);
5733                 return;
5734         }
5735
5736         for (i = 0; i < info->num_queue_pairs; i++) {
5737                 pair = &info->qpair[i];
5738
5739                 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
5740                     pair->rxq.vsi_id != vf->vsi.vsi_num ||
5741                     pair->txq.queue_id != pair->rxq.queue_id ||
5742                     pair->txq.queue_id >= vf->vsi.num_queues) {
5743
5744                         i40e_send_vf_nack(pf, vf,
5745                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5746                         return;
5747                 }
5748
5749                 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
5750                         i40e_send_vf_nack(pf, vf,
5751                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5752                         return;
5753                 }
5754
5755                 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
5756                         i40e_send_vf_nack(pf, vf,
5757                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5758                         return;
5759                 }
5760         }
5761
5762         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
5763 }
5764
5765 static void
5766 ixl_vf_set_qctl(struct ixl_pf *pf,
5767     const struct i40e_virtchnl_vector_map *vector,
5768     enum i40e_queue_type cur_type, uint16_t cur_queue,
5769     enum i40e_queue_type *last_type, uint16_t *last_queue)
5770 {
5771         uint32_t offset, qctl;
5772         uint16_t itr_indx;
5773
5774         if (cur_type == I40E_QUEUE_TYPE_RX) {
5775                 offset = I40E_QINT_RQCTL(cur_queue);
5776                 itr_indx = vector->rxitr_idx;
5777         } else {
5778                 offset = I40E_QINT_TQCTL(cur_queue);
5779                 itr_indx = vector->txitr_idx;
5780         }
5781
5782         qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
5783             (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
5784             (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
5785             I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5786             (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
5787
5788         wr32(&pf->hw, offset, qctl);
5789
5790         *last_type = cur_type;
5791         *last_queue = cur_queue;
5792 }
5793
5794 static void
5795 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
5796     const struct i40e_virtchnl_vector_map *vector)
5797 {
5798         struct i40e_hw *hw;
5799         u_int qindex;
5800         enum i40e_queue_type type, last_type;
5801         uint32_t lnklst_reg;
5802         uint16_t rxq_map, txq_map, cur_queue, last_queue;
5803
5804         hw = &pf->hw;
5805
5806         rxq_map = vector->rxq_map;
5807         txq_map = vector->txq_map;
5808
5809         last_queue = IXL_END_OF_INTR_LNKLST;
5810         last_type = I40E_QUEUE_TYPE_RX;
5811
5812         /*
5813          * The datasheet says to optimize performance, RX queues and TX queues
5814          * should be interleaved in the interrupt linked list, so we process
5815          * both at once here.
5816          */
5817         while ((rxq_map != 0) || (txq_map != 0)) {
5818                 if (txq_map != 0) {
5819                         qindex = ffs(txq_map) - 1;
5820                         type = I40E_QUEUE_TYPE_TX;
5821                         cur_queue = vf->vsi.first_queue + qindex;
5822                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
5823                             &last_type, &last_queue);
5824                         txq_map &= ~(1 << qindex);
5825                 }
5826
5827                 if (rxq_map != 0) {
5828                         qindex = ffs(rxq_map) - 1;
5829                         type = I40E_QUEUE_TYPE_RX;
5830                         cur_queue = vf->vsi.first_queue + qindex;
5831                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
5832                             &last_type, &last_queue);
5833                         rxq_map &= ~(1 << qindex);
5834                 }
5835         }
5836
5837         if (vector->vector_id == 0)
5838                 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
5839         else
5840                 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
5841                     vf->vf_num);
5842         wr32(hw, lnklst_reg,
5843             (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
5844             (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
5845
5846         ixl_flush(hw);
5847 }
5848
5849 static void
5850 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5851     uint16_t msg_size)
5852 {
5853         struct i40e_virtchnl_irq_map_info *map;
5854         struct i40e_virtchnl_vector_map *vector;
5855         struct i40e_hw *hw;
5856         int i, largest_txq, largest_rxq;
5857
5858         hw = &pf->hw;
5859
5860         if (msg_size < sizeof(*map)) {
5861                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5862                     I40E_ERR_PARAM);
5863                 return;
5864         }
5865
5866         map = msg;
5867         if (map->num_vectors == 0) {
5868                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5869                     I40E_ERR_PARAM);
5870                 return;
5871         }
5872
5873         if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
5874                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5875                     I40E_ERR_PARAM);
5876                 return;
5877         }
5878
5879         for (i = 0; i < map->num_vectors; i++) {
5880                 vector = &map->vecmap[i];
5881
5882                 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
5883                     vector->vsi_id != vf->vsi.vsi_num) {
5884                         i40e_send_vf_nack(pf, vf,
5885                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
5886                         return;
5887                 }
5888
5889                 if (vector->rxq_map != 0) {
5890                         largest_rxq = fls(vector->rxq_map) - 1;
5891                         if (largest_rxq >= vf->vsi.num_queues) {
5892                                 i40e_send_vf_nack(pf, vf,
5893                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5894                                     I40E_ERR_PARAM);
5895                                 return;
5896                         }
5897                 }
5898
5899                 if (vector->txq_map != 0) {
5900                         largest_txq = fls(vector->txq_map) - 1;
5901                         if (largest_txq >= vf->vsi.num_queues) {
5902                                 i40e_send_vf_nack(pf, vf,
5903                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5904                                     I40E_ERR_PARAM);
5905                                 return;
5906                         }
5907                 }
5908
5909                 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
5910                     vector->txitr_idx > IXL_MAX_ITR_IDX) {
5911                         i40e_send_vf_nack(pf, vf,
5912                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5913                             I40E_ERR_PARAM);
5914                         return;
5915                 }
5916
5917                 ixl_vf_config_vector(pf, vf, vector);
5918         }
5919
5920         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
5921 }
5922
5923 static void
5924 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5925     uint16_t msg_size)
5926 {
5927         struct i40e_virtchnl_queue_select *select;
5928         int error;
5929
5930         if (msg_size != sizeof(*select)) {
5931                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5932                     I40E_ERR_PARAM);
5933                 return;
5934         }
5935
5936         select = msg;
5937         if (select->vsi_id != vf->vsi.vsi_num ||
5938             select->rx_queues == 0 || select->tx_queues == 0) {
5939                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5940                     I40E_ERR_PARAM);
5941                 return;
5942         }
5943
5944         error = ixl_enable_rings(&vf->vsi);
5945         if (error) {
5946                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5947                     I40E_ERR_TIMEOUT);
5948                 return;
5949         }
5950
5951         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
5952 }
5953
5954 static void
5955 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
5956     void *msg, uint16_t msg_size)
5957 {
5958         struct i40e_virtchnl_queue_select *select;
5959         int error;
5960
5961         if (msg_size != sizeof(*select)) {
5962                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5963                     I40E_ERR_PARAM);
5964                 return;
5965         }
5966
5967         select = msg;
5968         if (select->vsi_id != vf->vsi.vsi_num ||
5969             select->rx_queues == 0 || select->tx_queues == 0) {
5970                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5971                     I40E_ERR_PARAM);
5972                 return;
5973         }
5974
5975         error = ixl_disable_rings(&vf->vsi);
5976         if (error) {
5977                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5978                     I40E_ERR_TIMEOUT);
5979                 return;
5980         }
5981
5982         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
5983 }
5984
5985 static boolean_t
5986 ixl_zero_mac(const uint8_t *addr)
5987 {
5988         uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
5989
5990         return (cmp_etheraddr(addr, zero));
5991 }
5992
5993 static boolean_t
5994 ixl_bcast_mac(const uint8_t *addr)
5995 {
5996
5997         return (cmp_etheraddr(addr, ixl_bcast_addr));
5998 }
5999
6000 static int
6001 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6002 {
6003
6004         if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6005                 return (EINVAL);
6006
6007         /*
6008          * If the VF is not allowed to change its MAC address, don't let it
6009          * set a MAC filter for an address that is not a multicast address and
6010          * is not its assigned MAC.
6011          */
6012         if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6013             !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6014                 return (EPERM);
6015
6016         return (0);
6017 }
6018
6019 static void
6020 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6021     uint16_t msg_size)
6022 {
6023         struct i40e_virtchnl_ether_addr_list *addr_list;
6024         struct i40e_virtchnl_ether_addr *addr;
6025         struct ixl_vsi *vsi;
6026         int i;
6027         size_t expected_size;
6028
6029         vsi = &vf->vsi;
6030
6031         if (msg_size < sizeof(*addr_list)) {
6032                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6033                     I40E_ERR_PARAM);
6034                 return;
6035         }
6036
6037         addr_list = msg;
6038         expected_size = sizeof(*addr_list) +
6039             addr_list->num_elements * sizeof(*addr);
6040
6041         if (addr_list->num_elements == 0 ||
6042             addr_list->vsi_id != vsi->vsi_num ||
6043             msg_size != expected_size) {
6044                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6045                     I40E_ERR_PARAM);
6046                 return;
6047         }
6048
6049         for (i = 0; i < addr_list->num_elements; i++) {
6050                 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6051                         i40e_send_vf_nack(pf, vf,
6052                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6053                         return;
6054                 }
6055         }
6056
6057         for (i = 0; i < addr_list->num_elements; i++) {
6058                 addr = &addr_list->list[i];
6059                 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6060         }
6061
6062         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6063 }
6064
6065 static void
6066 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6067     uint16_t msg_size)
6068 {
6069         struct i40e_virtchnl_ether_addr_list *addr_list;
6070         struct i40e_virtchnl_ether_addr *addr;
6071         size_t expected_size;
6072         int i;
6073
6074         if (msg_size < sizeof(*addr_list)) {
6075                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6076                     I40E_ERR_PARAM);
6077                 return;
6078         }
6079
6080         addr_list = msg;
6081         expected_size = sizeof(*addr_list) +
6082             addr_list->num_elements * sizeof(*addr);
6083
6084         if (addr_list->num_elements == 0 ||
6085             addr_list->vsi_id != vf->vsi.vsi_num ||
6086             msg_size != expected_size) {
6087                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6088                     I40E_ERR_PARAM);
6089                 return;
6090         }
6091
6092         for (i = 0; i < addr_list->num_elements; i++) {
6093                 addr = &addr_list->list[i];
6094                 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6095                         i40e_send_vf_nack(pf, vf,
6096                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6097                         return;
6098                 }
6099         }
6100
6101         for (i = 0; i < addr_list->num_elements; i++) {
6102                 addr = &addr_list->list[i];
6103                 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6104         }
6105
6106         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6107 }
6108
6109 static enum i40e_status_code
6110 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6111 {
6112         struct i40e_vsi_context vsi_ctx;
6113
6114         vsi_ctx.seid = vf->vsi.seid;
6115
6116         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6117         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6118         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6119             I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6120         return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6121 }
6122
6123 static void
6124 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6125     uint16_t msg_size)
6126 {
6127         struct i40e_virtchnl_vlan_filter_list *filter_list;
6128         enum i40e_status_code code;
6129         size_t expected_size;
6130         int i;
6131
6132         if (msg_size < sizeof(*filter_list)) {
6133                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6134                     I40E_ERR_PARAM);
6135                 return;
6136         }
6137
6138         filter_list = msg;
6139         expected_size = sizeof(*filter_list) +
6140             filter_list->num_elements * sizeof(uint16_t);
6141         if (filter_list->num_elements == 0 ||
6142             filter_list->vsi_id != vf->vsi.vsi_num ||
6143             msg_size != expected_size) {
6144                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6145                     I40E_ERR_PARAM);
6146                 return;
6147         }
6148
6149         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6150                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6151                     I40E_ERR_PARAM);
6152                 return;
6153         }
6154
6155         for (i = 0; i < filter_list->num_elements; i++) {
6156                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6157                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6158                             I40E_ERR_PARAM);
6159                         return;
6160                 }
6161         }
6162
6163         code = ixl_vf_enable_vlan_strip(pf, vf);
6164         if (code != I40E_SUCCESS) {
6165                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6166                     I40E_ERR_PARAM);
6167         }
6168
6169         for (i = 0; i < filter_list->num_elements; i++)
6170                 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6171
6172         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
6173 }
6174
6175 static void
6176 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6177     uint16_t msg_size)
6178 {
6179         struct i40e_virtchnl_vlan_filter_list *filter_list;
6180         int i;
6181         size_t expected_size;
6182
6183         if (msg_size < sizeof(*filter_list)) {
6184                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6185                     I40E_ERR_PARAM);
6186                 return;
6187         }
6188
6189         filter_list = msg;
6190         expected_size = sizeof(*filter_list) +
6191             filter_list->num_elements * sizeof(uint16_t);
6192         if (filter_list->num_elements == 0 ||
6193             filter_list->vsi_id != vf->vsi.vsi_num ||
6194             msg_size != expected_size) {
6195                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6196                     I40E_ERR_PARAM);
6197                 return;
6198         }
6199
6200         for (i = 0; i < filter_list->num_elements; i++) {
6201                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6202                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6203                             I40E_ERR_PARAM);
6204                         return;
6205                 }
6206         }
6207
6208         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6209                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6210                     I40E_ERR_PARAM);
6211                 return;
6212         }
6213
6214         for (i = 0; i < filter_list->num_elements; i++)
6215                 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6216
6217         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
6218 }
6219
6220 static void
6221 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6222     void *msg, uint16_t msg_size)
6223 {
6224         struct i40e_virtchnl_promisc_info *info;
6225         enum i40e_status_code code;
6226
6227         if (msg_size != sizeof(*info)) {
6228                 i40e_send_vf_nack(pf, vf,
6229                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6230                 return;
6231         }
6232
6233         if (!vf->vf_flags & VF_FLAG_PROMISC_CAP) {
6234                 i40e_send_vf_nack(pf, vf,
6235                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6236                 return;
6237         }
6238
6239         info = msg;
6240         if (info->vsi_id != vf->vsi.vsi_num) {
6241                 i40e_send_vf_nack(pf, vf,
6242                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6243                 return;
6244         }
6245
6246         code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
6247             info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
6248         if (code != I40E_SUCCESS) {
6249                 i40e_send_vf_nack(pf, vf,
6250                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6251                 return;
6252         }
6253
6254         code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
6255             info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
6256         if (code != I40E_SUCCESS) {
6257                 i40e_send_vf_nack(pf, vf,
6258                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6259                 return;
6260         }
6261
6262         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
6263 }
6264
6265 static void
6266 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6267     uint16_t msg_size)
6268 {
6269         struct i40e_virtchnl_queue_select *queue;
6270
6271         if (msg_size != sizeof(*queue)) {
6272                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6273                     I40E_ERR_PARAM);
6274                 return;
6275         }
6276
6277         queue = msg;
6278         if (queue->vsi_id != vf->vsi.vsi_num) {
6279                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6280                     I40E_ERR_PARAM);
6281                 return;
6282         }
6283
6284         ixl_update_eth_stats(&vf->vsi);
6285
6286         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6287             I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
6288 }
6289
6290 static void
6291 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
6292 {
6293         struct ixl_vf *vf;
6294         void *msg;
6295         uint16_t vf_num, msg_size;
6296         uint32_t opcode;
6297
6298         vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
6299         opcode = le32toh(event->desc.cookie_high);
6300
6301         if (vf_num >= pf->num_vfs) {
6302                 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
6303                 return;
6304         }
6305
6306         vf = &pf->vfs[vf_num];
6307         msg = event->msg_buf;
6308         msg_size = event->msg_len;
6309
6310         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
6311             "Got msg %s(%d) from VF-%d of size %d\n",
6312             ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
6313
6314         switch (opcode) {
6315         case I40E_VIRTCHNL_OP_VERSION:
6316                 ixl_vf_version_msg(pf, vf, msg, msg_size);
6317                 break;
6318         case I40E_VIRTCHNL_OP_RESET_VF:
6319                 ixl_vf_reset_msg(pf, vf, msg, msg_size);
6320                 break;
6321         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6322                 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
6323                 break;
6324         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6325                 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
6326                 break;
6327         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6328                 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
6329                 break;
6330         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6331                 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
6332                 break;
6333         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6334                 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
6335                 break;
6336         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6337                 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
6338                 break;
6339         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6340                 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
6341                 break;
6342         case I40E_VIRTCHNL_OP_ADD_VLAN:
6343                 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
6344                 break;
6345         case I40E_VIRTCHNL_OP_DEL_VLAN:
6346                 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
6347                 break;
6348         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6349                 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
6350                 break;
6351         case I40E_VIRTCHNL_OP_GET_STATS:
6352                 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
6353                 break;
6354
6355         /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
6356         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6357         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6358         default:
6359                 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
6360                 break;
6361         }
6362 }
6363
6364 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
6365 static void
6366 ixl_handle_vflr(void *arg, int pending)
6367 {
6368         struct ixl_pf *pf;
6369         struct i40e_hw *hw;
6370         uint16_t global_vf_num;
6371         uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
6372         int i;
6373
6374         pf = arg;
6375         hw = &pf->hw;
6376
6377         IXL_PF_LOCK(pf);
6378         for (i = 0; i < pf->num_vfs; i++) {
6379                 global_vf_num = hw->func_caps.vf_base_id + i;
6380
6381                 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
6382                 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
6383                 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
6384                 if (vflrstat & vflrstat_mask) {
6385                         wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
6386                             vflrstat_mask);
6387
6388                         ixl_reinit_vf(pf, &pf->vfs[i]);
6389                 }
6390         }
6391
6392         icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
6393         icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
6394         wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
6395         ixl_flush(hw);
6396
6397         IXL_PF_UNLOCK(pf);
6398 }
6399
6400 static int
6401 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
6402 {
6403
6404         switch (err) {
6405         case I40E_AQ_RC_EPERM:
6406                 return (EPERM);
6407         case I40E_AQ_RC_ENOENT:
6408                 return (ENOENT);
6409         case I40E_AQ_RC_ESRCH:
6410                 return (ESRCH);
6411         case I40E_AQ_RC_EINTR:
6412                 return (EINTR);
6413         case I40E_AQ_RC_EIO:
6414                 return (EIO);
6415         case I40E_AQ_RC_ENXIO:
6416                 return (ENXIO);
6417         case I40E_AQ_RC_E2BIG:
6418                 return (E2BIG);
6419         case I40E_AQ_RC_EAGAIN:
6420                 return (EAGAIN);
6421         case I40E_AQ_RC_ENOMEM:
6422                 return (ENOMEM);
6423         case I40E_AQ_RC_EACCES:
6424                 return (EACCES);
6425         case I40E_AQ_RC_EFAULT:
6426                 return (EFAULT);
6427         case I40E_AQ_RC_EBUSY:
6428                 return (EBUSY);
6429         case I40E_AQ_RC_EEXIST:
6430                 return (EEXIST);
6431         case I40E_AQ_RC_EINVAL:
6432                 return (EINVAL);
6433         case I40E_AQ_RC_ENOTTY:
6434                 return (ENOTTY);
6435         case I40E_AQ_RC_ENOSPC:
6436                 return (ENOSPC);
6437         case I40E_AQ_RC_ENOSYS:
6438                 return (ENOSYS);
6439         case I40E_AQ_RC_ERANGE:
6440                 return (ERANGE);
6441         case I40E_AQ_RC_EFLUSHED:
6442                 return (EINVAL);        /* No exact equivalent in errno.h */
6443         case I40E_AQ_RC_BAD_ADDR:
6444                 return (EFAULT);
6445         case I40E_AQ_RC_EMODE:
6446                 return (EPERM);
6447         case I40E_AQ_RC_EFBIG:
6448                 return (EFBIG);
6449         default:
6450                 return (EINVAL);
6451         }
6452 }
6453
6454 static int
6455 ixl_init_iov(device_t dev, uint16_t num_vfs, const nvlist_t *params)
6456 {
6457         struct ixl_pf *pf;
6458         struct i40e_hw *hw;
6459         struct ixl_vsi *pf_vsi;
6460         enum i40e_status_code ret;
6461         int i, error;
6462
6463         pf = device_get_softc(dev);
6464         hw = &pf->hw;
6465         pf_vsi = &pf->vsi;
6466
6467         IXL_PF_LOCK(pf);
6468         pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
6469             M_ZERO);
6470
6471         if (pf->vfs == NULL) {
6472                 error = ENOMEM;
6473                 goto fail;
6474         }
6475
6476         for (i = 0; i < num_vfs; i++)
6477                 sysctl_ctx_init(&pf->vfs[i].ctx);
6478
6479         ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
6480             1, FALSE, FALSE, &pf->veb_seid, NULL);
6481         if (ret != I40E_SUCCESS) {
6482                 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
6483                 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
6484                     error);
6485                 goto fail;
6486         }
6487
6488         ixl_configure_msix(pf);
6489         ixl_enable_adminq(hw);
6490
6491         pf->num_vfs = num_vfs;
6492         IXL_PF_UNLOCK(pf);
6493         return (0);
6494
6495 fail:
6496         free(pf->vfs, M_IXL);
6497         pf->vfs = NULL;
6498         IXL_PF_UNLOCK(pf);
6499         return (error);
6500 }
6501
6502 static void
6503 ixl_uninit_iov(device_t dev)
6504 {
6505         struct ixl_pf *pf;
6506         struct i40e_hw *hw;
6507         struct ixl_vsi *vsi;
6508         struct ifnet *ifp;
6509         struct ixl_vf *vfs;
6510         int i, num_vfs;
6511
6512         pf = device_get_softc(dev);
6513         hw = &pf->hw;
6514         vsi = &pf->vsi;
6515         ifp = vsi->ifp;
6516
6517         IXL_PF_LOCK(pf);
6518         for (i = 0; i < pf->num_vfs; i++) {
6519                 if (pf->vfs[i].vsi.seid != 0)
6520                         i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
6521         }
6522
6523         if (pf->veb_seid != 0) {
6524                 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
6525                 pf->veb_seid = 0;
6526         }
6527
6528         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
6529                 ixl_disable_intr(vsi);
6530
6531         vfs = pf->vfs;
6532         num_vfs = pf->num_vfs;
6533
6534         pf->vfs = NULL;
6535         pf->num_vfs = 0;
6536         IXL_PF_UNLOCK(pf);
6537
6538         /* Do this after the unlock as sysctl_ctx_free might sleep. */
6539         for (i = 0; i < num_vfs; i++)
6540                 sysctl_ctx_free(&vfs[i].ctx);
6541         free(vfs, M_IXL);
6542 }
6543
6544 static int
6545 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
6546 {
6547         char sysctl_name[QUEUE_NAME_LEN];
6548         struct ixl_pf *pf;
6549         struct ixl_vf *vf;
6550         const void *mac;
6551         size_t size;
6552         int error;
6553
6554         pf = device_get_softc(dev);
6555         vf = &pf->vfs[vfnum];
6556
6557         IXL_PF_LOCK(pf);
6558         vf->vf_num = vfnum;
6559
6560         vf->vsi.back = pf;
6561         vf->vf_flags = VF_FLAG_ENABLED;
6562         SLIST_INIT(&vf->vsi.ftl);
6563
6564         error = ixl_vf_setup_vsi(pf, vf);
6565         if (error != 0)
6566                 goto out;
6567
6568         if (nvlist_exists_binary(params, "mac-addr")) {
6569                 mac = nvlist_get_binary(params, "mac-addr", &size);
6570                 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
6571
6572                 if (nvlist_get_bool(params, "allow-set-mac"))
6573                         vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6574         } else
6575                 /*
6576                  * If the administrator has not specified a MAC address then
6577                  * we must allow the VF to choose one.
6578                  */
6579                 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6580
6581         if (nvlist_get_bool(params, "mac-anti-spoof"))
6582                 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
6583
6584         if (nvlist_get_bool(params, "allow-promisc"))
6585                 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
6586
6587         vf->vf_flags |= VF_FLAG_VLAN_CAP;
6588
6589         ixl_reset_vf(pf, vf);
6590 out:
6591         IXL_PF_UNLOCK(pf);
6592         if (error == 0) {
6593                 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
6594                 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
6595         }
6596
6597         return (error);
6598 }
6599 #endif /* PCI_IOV */