sys/dev/ixl/if_ixl.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifndef IXL_STANDALONE_BUILD
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #endif
  39
  40 #include "ixl.h"
  41 #include "ixl_pf.h"
  42
  43 #ifdef RSS
  44 #include <net/rss_config.h>
  45 #endif
  46
  47 /*********************************************************************
  48  *  Driver version
  49  *********************************************************************/
  50 char ixl_driver_version[] = "1.4.1";
  51
  52 /*********************************************************************
  53  *  PCI Device ID Table
  54  *
  55  *  Used by probe to select devices to load on
  56  *  Last field stores an index into ixl_strings
  57  *  Last entry must be all 0s
  58  *
  59  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  60  *********************************************************************/
  61
  62 static ixl_vendor_info_t ixl_vendor_info_array[] =
  63 {
  64         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
  65         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_A, 0, 0, 0},
  66         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
  67         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
  68         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
  69         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
  70         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
  71         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
  72         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2, 0, 0, 0},
  73         /* required last entry */
  74         {0, 0, 0, 0, 0}
  75 };
  76
  77 /*********************************************************************
  78  *  Table of branding strings
  79  *********************************************************************/
  80
  81 static char    *ixl_strings[] = {
  82         "Intel(R) Ethernet Connection XL710 Driver"
  83 };
  84
  85
  86 /*********************************************************************
  87  *  Function prototypes
  88  *********************************************************************/
  89 static int      ixl_probe(device_t);
  90 static int      ixl_attach(device_t);
  91 static int      ixl_detach(device_t);
  92 static int      ixl_shutdown(device_t);
  93 static int      ixl_get_hw_capabilities(struct ixl_pf *);
  94 static void     ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
  95 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
  96 static void     ixl_init(void *);
  97 static void     ixl_init_locked(struct ixl_pf *);
  98 static void     ixl_stop(struct ixl_pf *);
  99 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
 100 static int      ixl_media_change(struct ifnet *);
 101 static void     ixl_update_link_status(struct ixl_pf *);
 102 static int      ixl_allocate_pci_resources(struct ixl_pf *);
 103 static u16      ixl_get_bus_info(struct i40e_hw *, device_t);
 104 static int      ixl_setup_stations(struct ixl_pf *);
 105 static int      ixl_switch_config(struct ixl_pf *);
 106 static int      ixl_initialize_vsi(struct ixl_vsi *);
 107 static int      ixl_assign_vsi_msix(struct ixl_pf *);
 108 static int      ixl_assign_vsi_legacy(struct ixl_pf *);
 109 static int      ixl_init_msix(struct ixl_pf *);
 110 static void     ixl_configure_msix(struct ixl_pf *);
 111 static void     ixl_configure_itr(struct ixl_pf *);
 112 static void     ixl_configure_legacy(struct ixl_pf *);
 113 static void     ixl_free_pci_resources(struct ixl_pf *);
 114 static void     ixl_local_timer(void *);
 115 static int      ixl_setup_interface(device_t, struct ixl_vsi *);
 116 static void     ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
 117 static void     ixl_config_rss(struct ixl_vsi *);
 118 static void     ixl_set_queue_rx_itr(struct ixl_queue *);
 119 static void     ixl_set_queue_tx_itr(struct ixl_queue *);
 120 static int      ixl_set_advertised_speeds(struct ixl_pf *, int);
 121
 122 static int      ixl_enable_rings(struct ixl_vsi *);
 123 static int      ixl_disable_rings(struct ixl_vsi *);
 124 static void     ixl_enable_intr(struct ixl_vsi *);
 125 static void     ixl_disable_intr(struct ixl_vsi *);
 126 static void     ixl_disable_rings_intr(struct ixl_vsi *);
 127
 128 static void     ixl_enable_adminq(struct i40e_hw *);
 129 static void     ixl_disable_adminq(struct i40e_hw *);
 130 static void     ixl_enable_queue(struct i40e_hw *, int);
 131 static void     ixl_disable_queue(struct i40e_hw *, int);
 132 static void     ixl_enable_legacy(struct i40e_hw *);
 133 static void     ixl_disable_legacy(struct i40e_hw *);
 134
 135 static void     ixl_set_promisc(struct ixl_vsi *);
 136 static void     ixl_add_multi(struct ixl_vsi *);
 137 static void     ixl_del_multi(struct ixl_vsi *);
 138 static void     ixl_register_vlan(void *, struct ifnet *, u16);
 139 static void     ixl_unregister_vlan(void *, struct ifnet *, u16);
 140 static void     ixl_setup_vlan_filters(struct ixl_vsi *);
 141
 142 static void     ixl_init_filters(struct ixl_vsi *);
 143 static void     ixl_reconfigure_filters(struct ixl_vsi *vsi);
 144 static void     ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
 145 static void     ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
 146 static void     ixl_add_hw_filters(struct ixl_vsi *, int, int);
 147 static void     ixl_del_hw_filters(struct ixl_vsi *, int);
 148 static struct ixl_mac_filter *
 149                 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
 150 static void     ixl_add_mc_filter(struct ixl_vsi *, u8 *);
 151 static void     ixl_free_mac_filters(struct ixl_vsi *vsi);
 152
 153
 154 /* Sysctl debug interface */
 155 static int      ixl_debug_info(SYSCTL_HANDLER_ARGS);
 156 static void     ixl_print_debug_info(struct ixl_pf *);
 157
 158 /* The MSI/X Interrupt handlers */
 159 static void     ixl_intr(void *);
 160 static void     ixl_msix_que(void *);
 161 static void     ixl_msix_adminq(void *);
 162 static void     ixl_handle_mdd_event(struct ixl_pf *);
 163
 164 /* Deferred interrupt tasklets */
 165 static void     ixl_do_adminq(void *, int);
 166
 167 /* Sysctl handlers */
 168 static int      ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
 169 static int      ixl_set_advertise(SYSCTL_HANDLER_ARGS);
 170 static int      ixl_current_speed(SYSCTL_HANDLER_ARGS);
 171 static int      ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
 172
 173 /* Statistics */
 174 static void     ixl_add_hw_stats(struct ixl_pf *);
 175 static void     ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
 176                     struct sysctl_oid_list *, struct i40e_hw_port_stats *);
 177 static void     ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
 178                     struct sysctl_oid_list *,
 179                     struct i40e_eth_stats *);
 180 static void     ixl_update_stats_counters(struct ixl_pf *);
 181 static void     ixl_update_eth_stats(struct ixl_vsi *);
 182 static void     ixl_update_vsi_stats(struct ixl_vsi *);
 183 static void     ixl_pf_reset_stats(struct ixl_pf *);
 184 static void     ixl_vsi_reset_stats(struct ixl_vsi *);
 185 static void     ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
 186                     u64 *, u64 *);
 187 static void     ixl_stat_update32(struct i40e_hw *, u32, bool,
 188                     u64 *, u64 *);
 189
 190 #ifdef IXL_DEBUG_SYSCTL
 191 static int      ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
 192 static int      ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
 193 static int      ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 194 static int      ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
 195 static int      ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
 196 #endif
 197
 198 #ifdef PCI_IOV
 199 static int      ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
 200
 201 static int      ixl_init_iov(device_t dev, uint16_t num_vfs, const nvlist_t*);
 202 static void     ixl_uninit_iov(device_t dev);
 203 static int      ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
 204
 205 static void     ixl_handle_vf_msg(struct ixl_pf *,
 206                     struct i40e_arq_event_info *);
 207 static void     ixl_handle_vflr(void *arg, int pending);
 208
 209 static void     ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 210 static void     ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 211 #endif
 212
 213 /*********************************************************************
 214  *  FreeBSD Device Interface Entry Points
 215  *********************************************************************/
 216
 217 static device_method_t ixl_methods[] = {
 218         /* Device interface */
 219         DEVMETHOD(device_probe, ixl_probe),
 220         DEVMETHOD(device_attach, ixl_attach),
 221         DEVMETHOD(device_detach, ixl_detach),
 222         DEVMETHOD(device_shutdown, ixl_shutdown),
 223 #ifdef PCI_IOV
 224         DEVMETHOD(pci_init_iov, ixl_init_iov),
 225         DEVMETHOD(pci_uninit_iov, ixl_uninit_iov),
 226         DEVMETHOD(pci_add_vf, ixl_add_vf),
 227 #endif
 228         {0, 0}
 229 };
 230
 231 static driver_t ixl_driver = {
 232         "ixl", ixl_methods, sizeof(struct ixl_pf),
 233 };
 234
 235 devclass_t ixl_devclass;
 236 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
 237
 238 MODULE_DEPEND(ixl, pci, 1, 1, 1);
 239 MODULE_DEPEND(ixl, ether, 1, 1, 1);
 240
 241 #ifdef DEV_NETMAP
 242 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
 243 #endif /* DEV_NETMAP */
 244
 245
 246 /*
 247 ** Global reset mutex
 248 */
 249 static struct mtx ixl_reset_mtx;
 250
 251 /*
 252 ** TUNEABLE PARAMETERS:
 253 */
 254
 255 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
 256                    "IXL driver parameters");
 257
 258 /*
 259  * MSIX should be the default for best performance,
 260  * but this allows it to be forced off for testing.
 261  */
 262 static int ixl_enable_msix = 1;
 263 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
 264 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
 265     "Enable MSI-X interrupts");
 266
 267 /*
 268 ** Number of descriptors per ring:
 269 **   - TX and RX are the same size
 270 */
 271 static int ixl_ringsz = DEFAULT_RING;
 272 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
 273 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
 274     &ixl_ringsz, 0, "Descriptor Ring Size");
 275
 276 /*
 277 ** This can be set manually, if left as 0 the
 278 ** number of queues will be calculated based
 279 ** on cpus and msix vectors available.
 280 */
 281 int ixl_max_queues = 0;
 282 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
 283 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
 284     &ixl_max_queues, 0, "Number of Queues");
 285
 286 /*
 287 ** Controls for Interrupt Throttling
 288 **      - true/false for dynamic adjustment
 289 **      - default values for static ITR
 290 */
 291 int ixl_dynamic_rx_itr = 0;
 292 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
 293 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
 294     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
 295
 296 int ixl_dynamic_tx_itr = 0;
 297 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
 298 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
 299     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
 300
 301 int ixl_rx_itr = IXL_ITR_8K;
 302 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
 303 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
 304     &ixl_rx_itr, 0, "RX Interrupt Rate");
 305
 306 int ixl_tx_itr = IXL_ITR_4K;
 307 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
 308 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
 309     &ixl_tx_itr, 0, "TX Interrupt Rate");
 310
 311 #ifdef IXL_FDIR
 312 static int ixl_enable_fdir = 1;
 313 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
 314 /* Rate at which we sample */
 315 int ixl_atr_rate = 20;
 316 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
 317 #endif
 318
 319 #ifdef DEV_NETMAP
 320 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
 321 #include <dev/netmap/if_ixl_netmap.h>
 322 #endif /* DEV_NETMAP */
 323
 324 static char *ixl_fc_string[6] = {
 325         "None",
 326         "Rx",
 327         "Tx",
 328         "Full",
 329         "Priority",
 330         "Default"
 331 };
 332
 333 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
 334
 335 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
 336     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 337
 338 /*********************************************************************
 339  *  Device identification routine
 340  *
 341  *  ixl_probe determines if the driver should be loaded on
 342  *  the hardware based on PCI vendor/device id of the device.
 343  *
 344  *  return BUS_PROBE_DEFAULT on success, positive on failure
 345  *********************************************************************/
 346
 347 static int
 348 ixl_probe(device_t dev)
 349 {
 350         ixl_vendor_info_t *ent;
 351
 352         u16     pci_vendor_id, pci_device_id;
 353         u16     pci_subvendor_id, pci_subdevice_id;
 354         char    device_name[256];
 355         static bool lock_init = FALSE;
 356
 357         INIT_DEBUGOUT("ixl_probe: begin");
 358
 359         pci_vendor_id = pci_get_vendor(dev);
 360         if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
 361                 return (ENXIO);
 362
 363         pci_device_id = pci_get_device(dev);
 364         pci_subvendor_id = pci_get_subvendor(dev);
 365         pci_subdevice_id = pci_get_subdevice(dev);
 366
 367         ent = ixl_vendor_info_array;
 368         while (ent->vendor_id != 0) {
 369                 if ((pci_vendor_id == ent->vendor_id) &&
 370                     (pci_device_id == ent->device_id) &&
 371
 372                     ((pci_subvendor_id == ent->subvendor_id) ||
 373                      (ent->subvendor_id == 0)) &&
 374
 375                     ((pci_subdevice_id == ent->subdevice_id) ||
 376                      (ent->subdevice_id == 0))) {
 377                         sprintf(device_name, "%s, Version - %s",
 378                                 ixl_strings[ent->index],
 379                                 ixl_driver_version);
 380                         device_set_desc_copy(dev, device_name);
 381                         /* One shot mutex init */
 382                         if (lock_init == FALSE) {
 383                                 lock_init = TRUE;
 384                                 mtx_init(&ixl_reset_mtx,
 385                                     "ixl_reset",
 386                                     "IXL RESET Lock", MTX_DEF);
 387                         }
 388                         return (BUS_PROBE_DEFAULT);
 389                 }
 390                 ent++;
 391         }
 392         return (ENXIO);
 393 }
 394
 395 /*********************************************************************
 396  *  Device initialization routine
 397  *
 398  *  The attach entry point is called when the driver is being loaded.
 399  *  This routine identifies the type of hardware, allocates all resources
 400  *  and initializes the hardware.
 401  *
 402  *  return 0 on success, positive on failure
 403  *********************************************************************/
 404
 405 static int
 406 ixl_attach(device_t dev)
 407 {
 408         struct ixl_pf   *pf;
 409         struct i40e_hw  *hw;
 410         struct ixl_vsi *vsi;
 411         u16             bus;
 412         int             error = 0;
 413 #ifdef PCI_IOV
 414         nvlist_t        *pf_schema, *vf_schema;
 415         int             iov_error;
 416 #endif
 417
 418         INIT_DEBUGOUT("ixl_attach: begin");
 419
 420         /* Allocate, clear, and link in our primary soft structure */
 421         pf = device_get_softc(dev);
 422         pf->dev = pf->osdep.dev = dev;
 423         hw = &pf->hw;
 424
 425         /*
 426         ** Note this assumes we have a single embedded VSI,
 427         ** this could be enhanced later to allocate multiple
 428         */
 429         vsi = &pf->vsi;
 430         vsi->dev = pf->dev;
 431
 432         /* Core Lock Init*/
 433         IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
 434
 435         /* Set up the timer callout */
 436         callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
 437
 438         /* Set up sysctls */
 439         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 440             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 441             OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 442             pf, 0, ixl_set_flowcntl, "I", "Flow Control");
 443
 444         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 445             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 446             OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 447             pf, 0, ixl_set_advertise, "I", "Advertised Speed");
 448
 449         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 450             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 451             OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
 452             pf, 0, ixl_current_speed, "A", "Current Port Speed");
 453
 454         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 455             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 456             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
 457             pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
 458
 459         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 460             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 461             OID_AUTO, "rx_itr", CTLFLAG_RW,
 462             &ixl_rx_itr, IXL_ITR_8K, "RX ITR");
 463
 464         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 465             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 466             OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
 467             &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
 468
 469         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 470             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 471             OID_AUTO, "tx_itr", CTLFLAG_RW,
 472             &ixl_tx_itr, IXL_ITR_4K, "TX ITR");
 473
 474         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 475             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 476             OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
 477             &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
 478
 479 #ifdef IXL_DEBUG_SYSCTL
 480         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 481             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 482             OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
 483             pf, 0, ixl_sysctl_link_status, "A", "Current Link Status");
 484
 485         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 486             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 487             OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
 488             pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
 489
 490         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 491             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 492             OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
 493             pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
 494
 495         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 496             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 497             OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
 498             pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
 499
 500         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 501             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 502             OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
 503             pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
 504 #endif
 505
 506         /* Save off the PCI information */
 507         hw->vendor_id = pci_get_vendor(dev);
 508         hw->device_id = pci_get_device(dev);
 509         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
 510         hw->subsystem_vendor_id =
 511             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 512         hw->subsystem_device_id =
 513             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 514
 515         hw->bus.device = pci_get_slot(dev);
 516         hw->bus.func = pci_get_function(dev);
 517
 518         pf->vc_debug_lvl = 1;
 519
 520         /* Do PCI setup - map BAR0, etc */
 521         if (ixl_allocate_pci_resources(pf)) {
 522                 device_printf(dev, "Allocation of PCI resources failed\n");
 523                 error = ENXIO;
 524                 goto err_out;
 525         }
 526
 527         /* Create for initial debugging use */
 528         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 529             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 530             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
 531             ixl_debug_info, "I", "Debug Information");
 532
 533
 534         /* Establish a clean starting point */
 535         i40e_clear_hw(hw);
 536         error = i40e_pf_reset(hw);
 537         if (error) {
 538                 device_printf(dev,"PF reset failure %x\n", error);
 539                 error = EIO;
 540                 goto err_out;
 541         }
 542
 543         /* Set admin queue parameters */
 544         hw->aq.num_arq_entries = IXL_AQ_LEN;
 545         hw->aq.num_asq_entries = IXL_AQ_LEN;
 546         hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
 547         hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
 548
 549         /* Initialize the shared code */
 550         error = i40e_init_shared_code(hw);
 551         if (error) {
 552                 device_printf(dev,"Unable to initialize the shared code\n");
 553                 error = EIO;
 554                 goto err_out;
 555         }
 556
 557         /* Set up the admin queue */
 558         error = i40e_init_adminq(hw);
 559         if (error) {
 560                 device_printf(dev, "The driver for the device stopped "
 561                     "because the NVM image is newer than expected.\n"
 562                     "You must install the most recent version of "
 563                     " the network driver.\n");
 564                 goto err_out;
 565         }
 566         device_printf(dev, "%s\n", ixl_fw_version_str(hw));
 567
 568         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
 569             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
 570                 device_printf(dev, "The driver for the device detected "
 571                     "a newer version of the NVM image than expected.\n"
 572                     "Please install the most recent version of the network driver.\n");
 573         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
 574             hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
 575                 device_printf(dev, "The driver for the device detected "
 576                     "an older version of the NVM image than expected.\n"
 577                     "Please update the NVM image.\n");
 578
 579         /* Clear PXE mode */
 580         i40e_clear_pxe_mode(hw);
 581
 582         /* Get capabilities from the device */
 583         error = ixl_get_hw_capabilities(pf);
 584         if (error) {
 585                 device_printf(dev, "HW capabilities failure!\n");
 586                 goto err_get_cap;
 587         }
 588
 589         /* Set up host memory cache */
 590         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 591             hw->func_caps.num_rx_qp, 0, 0);
 592         if (error) {
 593                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
 594                 goto err_get_cap;
 595         }
 596
 597         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 598         if (error) {
 599                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
 600                 goto err_mac_hmc;
 601         }
 602
 603         /* Disable LLDP from the firmware */
 604         i40e_aq_stop_lldp(hw, TRUE, NULL);
 605
 606         i40e_get_mac_addr(hw, hw->mac.addr);
 607         error = i40e_validate_mac_addr(hw->mac.addr);
 608         if (error) {
 609                 device_printf(dev, "validate_mac_addr failed: %d\n", error);
 610                 goto err_mac_hmc;
 611         }
 612         bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
 613         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 614
 615         /* Set up VSI and queues */
 616         if (ixl_setup_stations(pf) != 0) {
 617                 device_printf(dev, "setup stations failed!\n");
 618                 error = ENOMEM;
 619                 goto err_mac_hmc;
 620         }
 621
 622         /* Initialize mac filter list for VSI */
 623         SLIST_INIT(&vsi->ftl);
 624
 625         /* Set up interrupt routing here */
 626         if (pf->msix > 1)
 627                 error = ixl_assign_vsi_msix(pf);
 628         else
 629                 error = ixl_assign_vsi_legacy(pf);
 630         if (error)
 631                 goto err_late;
 632
 633         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 634             (hw->aq.fw_maj_ver < 4)) {
 635                 i40e_msec_delay(75);
 636                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 637                 if (error)
 638                         device_printf(dev, "link restart failed, aq_err=%d\n",
 639                             pf->hw.aq.asq_last_status);
 640         }
 641
 642         /* Determine link state */
 643         i40e_aq_get_link_info(hw, TRUE, NULL, NULL);
 644         pf->link_up = i40e_get_link_status(hw);
 645
 646         /* Setup OS specific network interface */
 647         if (ixl_setup_interface(dev, vsi) != 0) {
 648                 device_printf(dev, "interface setup failed!\n");
 649                 error = EIO;
 650                 goto err_late;
 651         }
 652
 653         error = ixl_switch_config(pf);
 654         if (error) {
 655                 device_printf(dev, "Initial switch config failed: %d\n", error);
 656                 goto err_mac_hmc;
 657         }
 658
 659         /* Limit phy interrupts to link and modules failure */
 660         error = i40e_aq_set_phy_int_mask(hw,
 661             I40E_AQ_EVENT_LINK_UPDOWN | I40E_AQ_EVENT_MODULE_QUAL_FAIL, NULL);
 662         if (error)
 663                 device_printf(dev, "set phy mask failed: %d\n", error);
 664
 665         /* Get the bus configuration and set the shared code */
 666         bus = ixl_get_bus_info(hw, dev);
 667         i40e_set_pci_config_data(hw, bus);
 668
 669         /* Initialize statistics */
 670         ixl_pf_reset_stats(pf);
 671         ixl_update_stats_counters(pf);
 672         ixl_add_hw_stats(pf);
 673
 674         /* Register for VLAN events */
 675         vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 676             ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 677         vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 678             ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 679
 680 #ifdef PCI_IOV
 681         /* SR-IOV is only supported when MSI-X is in use. */
 682         if (pf->msix > 1) {
 683                 pf_schema = pci_iov_schema_alloc_node();
 684                 vf_schema = pci_iov_schema_alloc_node();
 685                 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
 686                 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
 687                     IOV_SCHEMA_HASDEFAULT, TRUE);
 688                 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
 689                     IOV_SCHEMA_HASDEFAULT, FALSE);
 690                 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
 691                     IOV_SCHEMA_HASDEFAULT, FALSE);
 692
 693                 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
 694                 if (iov_error != 0)
 695                         device_printf(dev,
 696                             "Failed to initialize SR-IOV (error=%d)\n",
 697                             iov_error);
 698         }
 699 #endif
 700
 701 #ifdef DEV_NETMAP
 702         ixl_netmap_attach(vsi);
 703 #endif /* DEV_NETMAP */
 704
 705         INIT_DEBUGOUT("ixl_attach: end");
 706         return (0);
 707
 708 err_late:
 709         if (vsi->ifp != NULL)
 710                 if_free(vsi->ifp);
 711 err_mac_hmc:
 712         i40e_shutdown_lan_hmc(hw);
 713 err_get_cap:
 714         i40e_shutdown_adminq(hw);
 715 err_out:
 716         ixl_free_pci_resources(pf);
 717         ixl_free_vsi(vsi);
 718         IXL_PF_LOCK_DESTROY(pf);
 719         return (error);
 720 }
 721
 722 /*********************************************************************
 723  *  Device removal routine
 724  *
 725  *  The detach entry point is called when the driver is being removed.
 726  *  This routine stops the adapter and deallocates all the resources
 727  *  that were allocated for driver operation.
 728  *
 729  *  return 0 on success, positive on failure
 730  *********************************************************************/
 731
 732 static int
 733 ixl_detach(device_t dev)
 734 {
 735         struct ixl_pf           *pf = device_get_softc(dev);
 736         struct i40e_hw          *hw = &pf->hw;
 737         struct ixl_vsi          *vsi = &pf->vsi;
 738         struct ixl_queue        *que = vsi->queues;
 739         i40e_status             status;
 740 #ifdef PCI_IOV
 741         int                     error;
 742 #endif
 743
 744         INIT_DEBUGOUT("ixl_detach: begin");
 745
 746         /* Make sure VLANS are not using driver */
 747         if (vsi->ifp->if_vlantrunk != NULL) {
 748                 device_printf(dev,"Vlan in use, detach first\n");
 749                 return (EBUSY);
 750         }
 751
 752 #ifdef PCI_IOV
 753         error = pci_iov_detach(dev);
 754         if (error != 0) {
 755                 device_printf(dev, "SR-IOV in use; detach first.\n");
 756                 return (error);
 757         }
 758 #endif
 759
 760         ether_ifdetach(vsi->ifp);
 761         if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) {
 762                 IXL_PF_LOCK(pf);
 763                 ixl_stop(pf);
 764                 IXL_PF_UNLOCK(pf);
 765         }
 766
 767         for (int i = 0; i < vsi->num_queues; i++, que++) {
 768                 if (que->tq) {
 769                         taskqueue_drain(que->tq, &que->task);
 770                         taskqueue_drain(que->tq, &que->tx_task);
 771                         taskqueue_free(que->tq);
 772                 }
 773         }
 774
 775         /* Shutdown LAN HMC */
 776         status = i40e_shutdown_lan_hmc(hw);
 777         if (status)
 778                 device_printf(dev,
 779                     "Shutdown LAN HMC failed with code %d\n", status);
 780
 781         /* Shutdown admin queue */
 782         status = i40e_shutdown_adminq(hw);
 783         if (status)
 784                 device_printf(dev,
 785                     "Shutdown Admin queue failed with code %d\n", status);
 786
 787         /* Unregister VLAN events */
 788         if (vsi->vlan_attach != NULL)
 789                 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
 790         if (vsi->vlan_detach != NULL)
 791                 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
 792
 793         callout_drain(&pf->timer);
 794 #ifdef DEV_NETMAP
 795         netmap_detach(vsi->ifp);
 796 #endif /* DEV_NETMAP */
 797         ixl_free_pci_resources(pf);
 798         bus_generic_detach(dev);
 799         if_free(vsi->ifp);
 800         ixl_free_vsi(vsi);
 801         IXL_PF_LOCK_DESTROY(pf);
 802         return (0);
 803 }
 804
 805 /*********************************************************************
 806  *
 807  *  Shutdown entry point
 808  *
 809  **********************************************************************/
 810
 811 static int
 812 ixl_shutdown(device_t dev)
 813 {
 814         struct ixl_pf *pf = device_get_softc(dev);
 815         IXL_PF_LOCK(pf);
 816         ixl_stop(pf);
 817         IXL_PF_UNLOCK(pf);
 818         return (0);
 819 }
 820
 821
 822 /*********************************************************************
 823  *
 824  *  Get the hardware capabilities
 825  *
 826  **********************************************************************/
 827
 828 static int
 829 ixl_get_hw_capabilities(struct ixl_pf *pf)
 830 {
 831         struct i40e_aqc_list_capabilities_element_resp *buf;
 832         struct i40e_hw  *hw = &pf->hw;
 833         device_t        dev = pf->dev;
 834         int             error, len;
 835         u16             needed;
 836         bool            again = TRUE;
 837
 838         len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
 839 retry:
 840         if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
 841             malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 842                 device_printf(dev, "Unable to allocate cap memory\n");
 843                 return (ENOMEM);
 844         }
 845
 846         /* This populates the hw struct */
 847         error = i40e_aq_discover_capabilities(hw, buf, len,
 848             &needed, i40e_aqc_opc_list_func_capabilities, NULL);
 849         free(buf, M_DEVBUF);
 850         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
 851             (again == TRUE)) {
 852                 /* retry once with a larger buffer */
 853                 again = FALSE;
 854                 len = needed;
 855                 goto retry;
 856         } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
 857                 device_printf(dev, "capability discovery failed: %d\n",
 858                     pf->hw.aq.asq_last_status);
 859                 return (ENODEV);
 860         }
 861
 862         /* Capture this PF's starting queue pair */
 863         pf->qbase = hw->func_caps.base_queue;
 864
 865 #ifdef IXL_DEBUG
 866         device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, "
 867             "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
 868             hw->pf_id, hw->func_caps.num_vfs,
 869             hw->func_caps.num_msix_vectors,
 870             hw->func_caps.num_msix_vectors_vf,
 871             hw->func_caps.fd_filters_guaranteed,
 872             hw->func_caps.fd_filters_best_effort,
 873             hw->func_caps.num_tx_qp,
 874             hw->func_caps.num_rx_qp,
 875             hw->func_caps.base_queue);
 876 #endif
 877         return (error);
 878 }
 879
 880 static void
 881 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
 882 {
 883         device_t        dev = vsi->dev;
 884
 885         /* Enable/disable TXCSUM/TSO4 */
 886         if (!(ifp->if_capenable & IFCAP_TXCSUM)
 887             && !(ifp->if_capenable & IFCAP_TSO4)) {
 888                 if (mask & IFCAP_TXCSUM) {
 889                         ifp->if_capenable |= IFCAP_TXCSUM;
 890                         /* enable TXCSUM, restore TSO if previously enabled */
 891                         if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
 892                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 893                                 ifp->if_capenable |= IFCAP_TSO4;
 894                         }
 895                 }
 896                 else if (mask & IFCAP_TSO4) {
 897                         ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
 898                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 899                         device_printf(dev,
 900                             "TSO4 requires txcsum, enabling both...\n");
 901                 }
 902         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 903             && !(ifp->if_capenable & IFCAP_TSO4)) {
 904                 if (mask & IFCAP_TXCSUM)
 905                         ifp->if_capenable &= ~IFCAP_TXCSUM;
 906                 else if (mask & IFCAP_TSO4)
 907                         ifp->if_capenable |= IFCAP_TSO4;
 908         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 909             && (ifp->if_capenable & IFCAP_TSO4)) {
 910                 if (mask & IFCAP_TXCSUM) {
 911                         vsi->flags |= IXL_FLAGS_KEEP_TSO4;
 912                         ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
 913                         device_printf(dev,
 914                             "TSO4 requires txcsum, disabling both...\n");
 915                 } else if (mask & IFCAP_TSO4)
 916                         ifp->if_capenable &= ~IFCAP_TSO4;
 917         }
 918
 919         /* Enable/disable TXCSUM_IPV6/TSO6 */
 920         if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 921             && !(ifp->if_capenable & IFCAP_TSO6)) {
 922                 if (mask & IFCAP_TXCSUM_IPV6) {
 923                         ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
 924                         if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
 925                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 926                                 ifp->if_capenable |= IFCAP_TSO6;
 927                         }
 928                 } else if (mask & IFCAP_TSO6) {
 929                         ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 930                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 931                         device_printf(dev,
 932                             "TSO6 requires txcsum6, enabling both...\n");
 933                 }
 934         } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 935             && !(ifp->if_capenable & IFCAP_TSO6)) {
 936                 if (mask & IFCAP_TXCSUM_IPV6)
 937                         ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
 938                 else if (mask & IFCAP_TSO6)
 939                         ifp->if_capenable |= IFCAP_TSO6;
 940         } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 941             && (ifp->if_capenable & IFCAP_TSO6)) {
 942                 if (mask & IFCAP_TXCSUM_IPV6) {
 943                         vsi->flags |= IXL_FLAGS_KEEP_TSO6;
 944                         ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 945                         device_printf(dev,
 946                             "TSO6 requires txcsum6, disabling both...\n");
 947                 } else if (mask & IFCAP_TSO6)
 948                         ifp->if_capenable &= ~IFCAP_TSO6;
 949         }
 950 }
 951
 952 /*********************************************************************
 953  *  Ioctl entry point
 954  *
 955  *  ixl_ioctl is called when the user wants to configure the
 956  *  interface.
 957  *
 958  *  return 0 on success, positive on failure
 959  **********************************************************************/
 960
 961 static int
 962 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 963 {
 964         struct ixl_vsi  *vsi = ifp->if_softc;
 965         struct ixl_pf   *pf = vsi->back;
 966         struct ifreq    *ifr = (struct ifreq *) data;
 967 #if defined(INET) || defined(INET6)
 968         struct ifaddr *ifa = (struct ifaddr *)data;
 969         bool            avoid_reset = FALSE;
 970 #endif
 971         int             error = 0;
 972
 973         switch (command) {
 974
 975         case SIOCSIFADDR:
 976 #ifdef INET
 977                 if (ifa->ifa_addr->sa_family == AF_INET)
 978                         avoid_reset = TRUE;
 979 #endif
 980 #ifdef INET6
 981                 if (ifa->ifa_addr->sa_family == AF_INET6)
 982                         avoid_reset = TRUE;
 983 #endif
 984 #if defined(INET) || defined(INET6)
 985                 /*
 986                 ** Calling init results in link renegotiation,
 987                 ** so we avoid doing it when possible.
 988                 */
 989                 if (avoid_reset) {
 990                         ifp->if_flags |= IFF_UP;
 991                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 992                                 ixl_init(pf);
 993 #ifdef INET
 994                         if (!(ifp->if_flags & IFF_NOARP))
 995                                 arp_ifinit(ifp, ifa);
 996 #endif
 997                 } else
 998                         error = ether_ioctl(ifp, command, data);
 999                 break;
1000 #endif
1001         case SIOCSIFMTU:
1002                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1003                 if (ifr->ifr_mtu > IXL_MAX_FRAME -
1004                    ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
1005                         error = EINVAL;
1006                 } else {
1007                         IXL_PF_LOCK(pf);
1008                         ifp->if_mtu = ifr->ifr_mtu;
1009                         vsi->max_frame_size =
1010                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
1011                             + ETHER_VLAN_ENCAP_LEN;
1012                         ixl_init_locked(pf);
1013                         IXL_PF_UNLOCK(pf);
1014                 }
1015                 break;
1016         case SIOCSIFFLAGS:
1017                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1018                 IXL_PF_LOCK(pf);
1019                 if (ifp->if_flags & IFF_UP) {
1020                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1021                                 if ((ifp->if_flags ^ pf->if_flags) &
1022                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1023                                         ixl_set_promisc(vsi);
1024                                 }
1025                         } else
1026                                 ixl_init_locked(pf);
1027                 } else
1028                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1029                                 ixl_stop(pf);
1030                 pf->if_flags = ifp->if_flags;
1031                 IXL_PF_UNLOCK(pf);
1032                 break;
1033         case SIOCADDMULTI:
1034                 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1035                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1036                         IXL_PF_LOCK(pf);
1037                         ixl_disable_intr(vsi);
1038                         ixl_add_multi(vsi);
1039                         ixl_enable_intr(vsi);
1040                         IXL_PF_UNLOCK(pf);
1041                 }
1042                 break;
1043         case SIOCDELMULTI:
1044                 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1045                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1046                         IXL_PF_LOCK(pf);
1047                         ixl_disable_intr(vsi);
1048                         ixl_del_multi(vsi);
1049                         ixl_enable_intr(vsi);
1050                         IXL_PF_UNLOCK(pf);
1051                 }
1052                 break;
1053         case SIOCSIFMEDIA:
1054         case SIOCGIFMEDIA:
1055                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1056                 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1057                 break;
1058         case SIOCSIFCAP:
1059         {
1060                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1061                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1062
1063                 ixl_cap_txcsum_tso(vsi, ifp, mask);
1064
1065                 if (mask & IFCAP_RXCSUM)
1066                         ifp->if_capenable ^= IFCAP_RXCSUM;
1067                 if (mask & IFCAP_RXCSUM_IPV6)
1068                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1069                 if (mask & IFCAP_LRO)
1070                         ifp->if_capenable ^= IFCAP_LRO;
1071                 if (mask & IFCAP_VLAN_HWTAGGING)
1072                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1073                 if (mask & IFCAP_VLAN_HWFILTER)
1074                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1075                 if (mask & IFCAP_VLAN_HWTSO)
1076                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1077                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1078                         IXL_PF_LOCK(pf);
1079                         ixl_init_locked(pf);
1080                         IXL_PF_UNLOCK(pf);
1081                 }
1082                 VLAN_CAPABILITIES(ifp);
1083
1084                 break;
1085         }
1086
1087         default:
1088                 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1089                 error = ether_ioctl(ifp, command, data);
1090                 break;
1091         }
1092
1093         return (error);
1094 }
1095
1096
1097 /*********************************************************************
1098  *  Init entry point
1099  *
1100  *  This routine is used in two ways. It is used by the stack as
1101  *  init entry point in network interface structure. It is also used
1102  *  by the driver as a hw/sw initialization routine to get to a
1103  *  consistent state.
1104  *
1105  *  return 0 on success, positive on failure
1106  **********************************************************************/
1107
1108 static void
1109 ixl_init_locked(struct ixl_pf *pf)
1110 {
1111         struct i40e_hw  *hw = &pf->hw;
1112         struct ixl_vsi  *vsi = &pf->vsi;
1113         struct ifnet    *ifp = vsi->ifp;
1114         device_t        dev = pf->dev;
1115         struct i40e_filter_control_settings     filter;
1116         u8              tmpaddr[ETHER_ADDR_LEN];
1117         int             ret;
1118
1119         mtx_assert(&pf->pf_mtx, MA_OWNED);
1120         INIT_DEBUGOUT("ixl_init: begin");
1121         ixl_stop(pf);
1122
1123         /* Get the latest mac address... User might use a LAA */
1124         bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1125               I40E_ETH_LENGTH_OF_ADDRESS);
1126         if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1127             i40e_validate_mac_addr(tmpaddr)) {
1128                 bcopy(tmpaddr, hw->mac.addr,
1129                     I40E_ETH_LENGTH_OF_ADDRESS);
1130                 ret = i40e_aq_mac_address_write(hw,
1131                     I40E_AQC_WRITE_TYPE_LAA_ONLY,
1132                     hw->mac.addr, NULL);
1133                 if (ret) {
1134                         device_printf(dev, "LLA address"
1135                          "change failed!!\n");
1136                         return;
1137                 }
1138         }
1139
1140         /* Set the various hardware offload abilities */
1141         ifp->if_hwassist = 0;
1142         if (ifp->if_capenable & IFCAP_TSO)
1143                 ifp->if_hwassist |= CSUM_TSO;
1144         if (ifp->if_capenable & IFCAP_TXCSUM)
1145                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1146         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1147                 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1148
1149         /* Set up the device filtering */
1150         bzero(&filter, sizeof(filter));
1151         filter.enable_ethtype = TRUE;
1152         filter.enable_macvlan = TRUE;
1153 #ifdef IXL_FDIR
1154         filter.enable_fdir = TRUE;
1155 #endif
1156         if (i40e_set_filter_control(hw, &filter))
1157                 device_printf(dev, "set_filter_control() failed\n");
1158
1159         /* Set up RSS */
1160         ixl_config_rss(vsi);
1161
1162         /*
1163         ** Prepare the VSI: rings, hmc contexts, etc...
1164         */
1165         if (ixl_initialize_vsi(vsi)) {
1166                 device_printf(dev, "initialize vsi failed!!\n");
1167                 return;
1168         }
1169
1170         /* Add protocol filters to list */
1171         ixl_init_filters(vsi);
1172
1173         /* Setup vlan's if needed */
1174         ixl_setup_vlan_filters(vsi);
1175
1176         /* Start the local timer */
1177         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1178
1179         /* Set up MSI/X routing and the ITR settings */
1180         if (ixl_enable_msix) {
1181                 ixl_configure_msix(pf);
1182                 ixl_configure_itr(pf);
1183         } else
1184                 ixl_configure_legacy(pf);
1185
1186         ixl_enable_rings(vsi);
1187
1188         i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1189
1190         ixl_reconfigure_filters(vsi);
1191
1192         /* Set MTU in hardware*/
1193         int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size,
1194             TRUE, 0, NULL);
1195         if (aq_error)
1196                 device_printf(vsi->dev,
1197                         "aq_set_mac_config in init error, code %d\n",
1198                     aq_error);
1199
1200         /* And now turn on interrupts */
1201         ixl_enable_intr(vsi);
1202
1203         /* Now inform the stack we're ready */
1204         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1205         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1206
1207         return;
1208 }
1209
1210 static void
1211 ixl_init(void *arg)
1212 {
1213         struct ixl_pf *pf = arg;
1214
1215         IXL_PF_LOCK(pf);
1216         ixl_init_locked(pf);
1217         IXL_PF_UNLOCK(pf);
1218         return;
1219 }
1220
1221 /*
1222 **
1223 ** MSIX Interrupt Handlers and Tasklets
1224 **
1225 */
1226 static void
1227 ixl_handle_que(void *context, int pending)
1228 {
1229         struct ixl_queue *que = context;
1230         struct ixl_vsi *vsi = que->vsi;
1231         struct i40e_hw  *hw = vsi->hw;
1232         struct tx_ring  *txr = &que->txr;
1233         struct ifnet    *ifp = vsi->ifp;
1234         bool            more;
1235
1236         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1237                 more = ixl_rxeof(que, IXL_RX_LIMIT);
1238                 IXL_TX_LOCK(txr);
1239                 ixl_txeof(que);
1240                 if (!drbr_empty(ifp, txr->br))
1241                         ixl_mq_start_locked(ifp, txr);
1242                 IXL_TX_UNLOCK(txr);
1243                 if (more) {
1244                         taskqueue_enqueue(que->tq, &que->task);
1245                         return;
1246                 }
1247         }
1248
1249         /* Reenable this interrupt - hmmm */
1250         ixl_enable_queue(hw, que->me);
1251         return;
1252 }
1253
1254
1255 /*********************************************************************
1256  *
1257  *  Legacy Interrupt Service routine
1258  *
1259  **********************************************************************/
1260 void
1261 ixl_intr(void *arg)
1262 {
1263         struct ixl_pf           *pf = arg;
1264         struct i40e_hw          *hw =  &pf->hw;
1265         struct ixl_vsi          *vsi = &pf->vsi;
1266         struct ixl_queue        *que = vsi->queues;
1267         struct ifnet            *ifp = vsi->ifp;
1268         struct tx_ring          *txr = &que->txr;
1269         u32                     reg, icr0, mask;
1270         bool                    more_tx, more_rx;
1271
1272         ++que->irqs;
1273
1274         /* Protect against spurious interrupts */
1275         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1276                 return;
1277
1278         icr0 = rd32(hw, I40E_PFINT_ICR0);
1279
1280         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1281         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1282         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1283
1284         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1285
1286 #ifdef PCI_IOV
1287         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1288                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1289 #endif
1290
1291         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1292                 taskqueue_enqueue(pf->tq, &pf->adminq);
1293                 return;
1294         }
1295
1296         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1297
1298         IXL_TX_LOCK(txr);
1299         more_tx = ixl_txeof(que);
1300         if (!drbr_empty(vsi->ifp, txr->br))
1301                 more_tx = 1;
1302         IXL_TX_UNLOCK(txr);
1303
1304         /* re-enable other interrupt causes */
1305         wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1306
1307         /* And now the queues */
1308         reg = rd32(hw, I40E_QINT_RQCTL(0));
1309         reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1310         wr32(hw, I40E_QINT_RQCTL(0), reg);
1311
1312         reg = rd32(hw, I40E_QINT_TQCTL(0));
1313         reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1314         reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1315         wr32(hw, I40E_QINT_TQCTL(0), reg);
1316
1317         ixl_enable_legacy(hw);
1318
1319         return;
1320 }
1321
1322
1323 /*********************************************************************
1324  *
1325  *  MSIX VSI Interrupt Service routine
1326  *
1327  **********************************************************************/
1328 void
1329 ixl_msix_que(void *arg)
1330 {
1331         struct ixl_queue        *que = arg;
1332         struct ixl_vsi  *vsi = que->vsi;
1333         struct i40e_hw  *hw = vsi->hw;
1334         struct tx_ring  *txr = &que->txr;
1335         bool            more_tx, more_rx;
1336
1337         /* Protect against spurious interrupts */
1338         if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1339                 return;
1340
1341         ++que->irqs;
1342
1343         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1344
1345         IXL_TX_LOCK(txr);
1346         more_tx = ixl_txeof(que);
1347         /*
1348         ** Make certain that if the stack
1349         ** has anything queued the task gets
1350         ** scheduled to handle it.
1351         */
1352         if (!drbr_empty(vsi->ifp, txr->br))
1353                 more_tx = 1;
1354         IXL_TX_UNLOCK(txr);
1355
1356         ixl_set_queue_rx_itr(que);
1357         ixl_set_queue_tx_itr(que);
1358
1359         if (more_tx || more_rx)
1360                 taskqueue_enqueue(que->tq, &que->task);
1361         else
1362                 ixl_enable_queue(hw, que->me);
1363
1364         return;
1365 }
1366
1367
1368 /*********************************************************************
1369  *
1370  *  MSIX Admin Queue Interrupt Service routine
1371  *
1372  **********************************************************************/
1373 static void
1374 ixl_msix_adminq(void *arg)
1375 {
1376         struct ixl_pf   *pf = arg;
1377         struct i40e_hw  *hw = &pf->hw;
1378         u32             reg, mask;
1379
1380         ++pf->admin_irq;
1381
1382         reg = rd32(hw, I40E_PFINT_ICR0);
1383         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1384
1385         /* Check on the cause */
1386         if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
1387                 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
1388
1389         if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1390                 ixl_handle_mdd_event(pf);
1391                 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
1392         }
1393
1394 #ifdef PCI_IOV
1395         if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1396                 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1397                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1398         }
1399 #endif
1400
1401         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1402         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1403         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1404
1405         taskqueue_enqueue(pf->tq, &pf->adminq);
1406         return;
1407 }
1408
1409 /*********************************************************************
1410  *
1411  *  Media Ioctl callback
1412  *
1413  *  This routine is called whenever the user queries the status of
1414  *  the interface using ifconfig.
1415  *
1416  **********************************************************************/
1417 static void
1418 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1419 {
1420         struct ixl_vsi  *vsi = ifp->if_softc;
1421         struct ixl_pf   *pf = vsi->back;
1422         struct i40e_hw  *hw = &pf->hw;
1423
1424         INIT_DEBUGOUT("ixl_media_status: begin");
1425         IXL_PF_LOCK(pf);
1426
1427         hw->phy.get_link_info = TRUE;
1428         pf->link_up = i40e_get_link_status(hw);
1429         ixl_update_link_status(pf);
1430
1431         ifmr->ifm_status = IFM_AVALID;
1432         ifmr->ifm_active = IFM_ETHER;
1433
1434         if (!pf->link_up) {
1435                 IXL_PF_UNLOCK(pf);
1436                 return;
1437         }
1438
1439         ifmr->ifm_status |= IFM_ACTIVE;
1440         /* Hardware is always full-duplex */
1441         ifmr->ifm_active |= IFM_FDX;
1442
1443         switch (hw->phy.link_info.phy_type) {
1444                 /* 100 M */
1445                 case I40E_PHY_TYPE_100BASE_TX:
1446                         ifmr->ifm_active |= IFM_100_TX;
1447                         break;
1448                 /* 1 G */
1449                 case I40E_PHY_TYPE_1000BASE_T:
1450                         ifmr->ifm_active |= IFM_1000_T;
1451                         break;
1452                 case I40E_PHY_TYPE_1000BASE_SX:
1453                         ifmr->ifm_active |= IFM_1000_SX;
1454                         break;
1455                 case I40E_PHY_TYPE_1000BASE_LX:
1456                         ifmr->ifm_active |= IFM_1000_LX;
1457                         break;
1458                 /* 10 G */
1459                 case I40E_PHY_TYPE_10GBASE_CR1:
1460                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1461                 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1462                 /* Using this until a real KR media type */
1463                 case I40E_PHY_TYPE_10GBASE_KR:
1464                 case I40E_PHY_TYPE_10GBASE_KX4:
1465                         ifmr->ifm_active |= IFM_10G_TWINAX;
1466                         break;
1467                 case I40E_PHY_TYPE_10GBASE_SR:
1468                         ifmr->ifm_active |= IFM_10G_SR;
1469                         break;
1470                 case I40E_PHY_TYPE_10GBASE_LR:
1471                         ifmr->ifm_active |= IFM_10G_LR;
1472                         break;
1473                 case I40E_PHY_TYPE_10GBASE_T:
1474                         ifmr->ifm_active |= IFM_10G_T;
1475                         break;
1476                 /* 40 G */
1477                 case I40E_PHY_TYPE_40GBASE_CR4:
1478                 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1479                         ifmr->ifm_active |= IFM_40G_CR4;
1480                         break;
1481                 case I40E_PHY_TYPE_40GBASE_SR4:
1482                         ifmr->ifm_active |= IFM_40G_SR4;
1483                         break;
1484                 case I40E_PHY_TYPE_40GBASE_LR4:
1485                         ifmr->ifm_active |= IFM_40G_LR4;
1486                         break;
1487                 /*
1488                 ** Set these to CR4 because OS does not
1489                 ** have types available yet.
1490                 */
1491                 case I40E_PHY_TYPE_40GBASE_KR4:
1492                 case I40E_PHY_TYPE_XLAUI:
1493                 case I40E_PHY_TYPE_XLPPI:
1494                 case I40E_PHY_TYPE_40GBASE_AOC:
1495                         ifmr->ifm_active |= IFM_40G_CR4;
1496                         break;
1497                 default:
1498                         ifmr->ifm_active |= IFM_UNKNOWN;
1499                         break;
1500         }
1501         /* Report flow control status as well */
1502         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1503                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1504         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1505                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1506
1507         IXL_PF_UNLOCK(pf);
1508
1509         return;
1510 }
1511
1512 /*********************************************************************
1513  *
1514  *  Media Ioctl callback
1515  *
1516  *  This routine is called when the user changes speed/duplex using
1517  *  media/mediopt option with ifconfig.
1518  *
1519  **********************************************************************/
1520 static int
1521 ixl_media_change(struct ifnet * ifp)
1522 {
1523         struct ixl_vsi *vsi = ifp->if_softc;
1524         struct ifmedia *ifm = &vsi->media;
1525
1526         INIT_DEBUGOUT("ixl_media_change: begin");
1527
1528         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1529                 return (EINVAL);
1530
1531         if_printf(ifp, "Media change is currently not supported.\n");
1532
1533         return (ENODEV);
1534 }
1535
1536
1537 #ifdef IXL_FDIR
1538 /*
1539 ** ATR: Application Targetted Receive - creates a filter
1540 **      based on TX flow info that will keep the receive
1541 **      portion of the flow on the same queue. Based on the
1542 **      implementation this is only available for TCP connections
1543 */
1544 void
1545 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1546 {
1547         struct ixl_vsi                  *vsi = que->vsi;
1548         struct tx_ring                  *txr = &que->txr;
1549         struct i40e_filter_program_desc *FDIR;
1550         u32                             ptype, dtype;
1551         int                             idx;
1552
1553         /* check if ATR is enabled and sample rate */
1554         if ((!ixl_enable_fdir) || (!txr->atr_rate))
1555                 return;
1556         /*
1557         ** We sample all TCP SYN/FIN packets,
1558         ** or at the selected sample rate
1559         */
1560         txr->atr_count++;
1561         if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1562             (txr->atr_count < txr->atr_rate))
1563                 return;
1564         txr->atr_count = 0;
1565
1566         /* Get a descriptor to use */
1567         idx = txr->next_avail;
1568         FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1569         if (++idx == que->num_desc)
1570                 idx = 0;
1571         txr->avail--;
1572         txr->next_avail = idx;
1573
1574         ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1575             I40E_TXD_FLTR_QW0_QINDEX_MASK;
1576
1577         ptype |= (etype == ETHERTYPE_IP) ?
1578             (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1579             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1580             (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1581             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1582
1583         ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1584
1585         dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1586
1587         /*
1588         ** We use the TCP TH_FIN as a trigger to remove
1589         ** the filter, otherwise its an update.
1590         */
1591         dtype |= (th->th_flags & TH_FIN) ?
1592             (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1593             I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1594             (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1595             I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1596
1597         dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1598             I40E_TXD_FLTR_QW1_DEST_SHIFT;
1599
1600         dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1601             I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1602
1603         FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1604         FDIR->dtype_cmd_cntindex = htole32(dtype);
1605         return;
1606 }
1607 #endif
1608
1609
1610 static void
1611 ixl_set_promisc(struct ixl_vsi *vsi)
1612 {
1613         struct ifnet    *ifp = vsi->ifp;
1614         struct i40e_hw  *hw = vsi->hw;
1615         int             err, mcnt = 0;
1616         bool            uni = FALSE, multi = FALSE;
1617
1618         if (ifp->if_flags & IFF_ALLMULTI)
1619                 multi = TRUE;
1620         else { /* Need to count the multicast addresses */
1621                 struct  ifmultiaddr *ifma;
1622                 if_maddr_rlock(ifp);
1623                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1624                         if (ifma->ifma_addr->sa_family != AF_LINK)
1625                                 continue;
1626                         if (mcnt == MAX_MULTICAST_ADDR)
1627                                 break;
1628                         mcnt++;
1629                 }
1630                 if_maddr_runlock(ifp);
1631         }
1632
1633         if (mcnt >= MAX_MULTICAST_ADDR)
1634                 multi = TRUE;
1635         if (ifp->if_flags & IFF_PROMISC)
1636                 uni = TRUE;
1637
1638         err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1639             vsi->seid, uni, NULL);
1640         err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1641             vsi->seid, multi, NULL);
1642         return;
1643 }
1644
1645 /*********************************************************************
1646  *      Filter Routines
1647  *
1648  *      Routines for multicast and vlan filter management.
1649  *
1650  *********************************************************************/
1651 static void
1652 ixl_add_multi(struct ixl_vsi *vsi)
1653 {
1654         struct  ifmultiaddr     *ifma;
1655         struct ifnet            *ifp = vsi->ifp;
1656         struct i40e_hw          *hw = vsi->hw;
1657         int                     mcnt = 0, flags;
1658
1659         IOCTL_DEBUGOUT("ixl_add_multi: begin");
1660
1661         if_maddr_rlock(ifp);
1662         /*
1663         ** First just get a count, to decide if we
1664         ** we simply use multicast promiscuous.
1665         */
1666         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1667                 if (ifma->ifma_addr->sa_family != AF_LINK)
1668                         continue;
1669                 mcnt++;
1670         }
1671         if_maddr_runlock(ifp);
1672
1673         if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1674                 /* delete existing MC filters */
1675                 ixl_del_hw_filters(vsi, mcnt);
1676                 i40e_aq_set_vsi_multicast_promiscuous(hw,
1677                     vsi->seid, TRUE, NULL);
1678                 return;
1679         }
1680
1681         mcnt = 0;
1682         if_maddr_rlock(ifp);
1683         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1684                 if (ifma->ifma_addr->sa_family != AF_LINK)
1685                         continue;
1686                 ixl_add_mc_filter(vsi,
1687                     (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1688                 mcnt++;
1689         }
1690         if_maddr_runlock(ifp);
1691         if (mcnt > 0) {
1692                 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1693                 ixl_add_hw_filters(vsi, flags, mcnt);
1694         }
1695
1696         IOCTL_DEBUGOUT("ixl_add_multi: end");
1697         return;
1698 }
1699
1700 static void
1701 ixl_del_multi(struct ixl_vsi *vsi)
1702 {
1703         struct ifnet            *ifp = vsi->ifp;
1704         struct ifmultiaddr      *ifma;
1705         struct ixl_mac_filter   *f;
1706         int                     mcnt = 0;
1707         bool            match = FALSE;
1708
1709         IOCTL_DEBUGOUT("ixl_del_multi: begin");
1710
1711         /* Search for removed multicast addresses */
1712         if_maddr_rlock(ifp);
1713         SLIST_FOREACH(f, &vsi->ftl, next) {
1714                 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1715                         match = FALSE;
1716                         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1717                                 if (ifma->ifma_addr->sa_family != AF_LINK)
1718                                         continue;
1719                                 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
1720                                 if (cmp_etheraddr(f->macaddr, mc_addr)) {
1721                                         match = TRUE;
1722                                         break;
1723                                 }
1724                         }
1725                         if (match == FALSE) {
1726                                 f->flags |= IXL_FILTER_DEL;
1727                                 mcnt++;
1728                         }
1729                 }
1730         }
1731         if_maddr_runlock(ifp);
1732
1733         if (mcnt > 0)
1734                 ixl_del_hw_filters(vsi, mcnt);
1735 }
1736
1737
1738 /*********************************************************************
1739  *  Timer routine
1740  *
1741  *  This routine checks for link status,updates statistics,
1742  *  and runs the watchdog check.
1743  *
1744  **********************************************************************/
1745
1746 static void
1747 ixl_local_timer(void *arg)
1748 {
1749         struct ixl_pf           *pf = arg;
1750         struct i40e_hw          *hw = &pf->hw;
1751         struct ixl_vsi          *vsi = &pf->vsi;
1752         struct ixl_queue        *que = vsi->queues;
1753         device_t                dev = pf->dev;
1754         int                     hung = 0;
1755         u32                     mask;
1756
1757         mtx_assert(&pf->pf_mtx, MA_OWNED);
1758
1759         /* Fire off the adminq task */
1760         taskqueue_enqueue(pf->tq, &pf->adminq);
1761
1762         /* Update stats */
1763         ixl_update_stats_counters(pf);
1764
1765         /*
1766         ** Check status of the queues
1767         */
1768         mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
1769                 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
1770
1771         for (int i = 0; i < vsi->num_queues; i++,que++) {
1772                 /* Any queues with outstanding work get a sw irq */
1773                 if (que->busy)
1774                         wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
1775                 /*
1776                 ** Each time txeof runs without cleaning, but there
1777                 ** are uncleaned descriptors it increments busy. If
1778                 ** we get to 5 we declare it hung.
1779                 */
1780                 if (que->busy == IXL_QUEUE_HUNG) {
1781                         ++hung;
1782                         /* Mark the queue as inactive */
1783                         vsi->active_queues &= ~((u64)1 << que->me);
1784                         continue;
1785                 } else {
1786                         /* Check if we've come back from hung */
1787                         if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
1788                                 vsi->active_queues |= ((u64)1 << que->me);
1789                 }
1790                 if (que->busy >= IXL_MAX_TX_BUSY) {
1791 #ifdef IXL_DEBUG
1792                         device_printf(dev,"Warning queue %d "
1793                             "appears to be hung!\n", i);
1794 #endif
1795                         que->busy = IXL_QUEUE_HUNG;
1796                         ++hung;
1797                 }
1798         }
1799         /* Only reinit if all queues show hung */
1800         if (hung == vsi->num_queues)
1801                 goto hung;
1802
1803         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1804         return;
1805
1806 hung:
1807         device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
1808         ixl_init_locked(pf);
1809 }
1810
1811 /*
1812 ** Note: this routine updates the OS on the link state
1813 **      the real check of the hardware only happens with
1814 **      a link interrupt.
1815 */
1816 static void
1817 ixl_update_link_status(struct ixl_pf *pf)
1818 {
1819         struct ixl_vsi          *vsi = &pf->vsi;
1820         struct i40e_hw          *hw = &pf->hw;
1821         struct ifnet            *ifp = vsi->ifp;
1822         device_t                dev = pf->dev;
1823
1824         if (pf->link_up){
1825                 if (vsi->link_active == FALSE) {
1826                         pf->fc = hw->fc.current_mode;
1827                         if (bootverbose) {
1828                                 device_printf(dev,"Link is up %d Gbps %s,"
1829                                     " Flow Control: %s\n",
1830                                     ((pf->link_speed ==
1831                                     I40E_LINK_SPEED_40GB)? 40:10),
1832                                     "Full Duplex", ixl_fc_string[pf->fc]);
1833                         }
1834                         vsi->link_active = TRUE;
1835                         /*
1836                         ** Warn user if link speed on NPAR enabled
1837                         ** partition is not at least 10GB
1838                         */
1839                         if (hw->func_caps.npar_enable &&
1840                            (hw->phy.link_info.link_speed ==
1841                            I40E_LINK_SPEED_1GB ||
1842                            hw->phy.link_info.link_speed ==
1843                            I40E_LINK_SPEED_100MB))
1844                                 device_printf(dev, "The partition detected"
1845                                     "link speed that is less than 10Gbps\n");
1846                         if_link_state_change(ifp, LINK_STATE_UP);
1847                 }
1848         } else { /* Link down */
1849                 if (vsi->link_active == TRUE) {
1850                         if (bootverbose)
1851                                 device_printf(dev,"Link is Down\n");
1852                         if_link_state_change(ifp, LINK_STATE_DOWN);
1853                         vsi->link_active = FALSE;
1854                 }
1855         }
1856
1857         return;
1858 }
1859
1860 /*********************************************************************
1861  *
1862  *  This routine disables all traffic on the adapter by issuing a
1863  *  global reset on the MAC and deallocates TX/RX buffers.
1864  *
1865  **********************************************************************/
1866
1867 static void
1868 ixl_stop(struct ixl_pf *pf)
1869 {
1870         struct ixl_vsi  *vsi = &pf->vsi;
1871         struct ifnet    *ifp = vsi->ifp;
1872
1873         mtx_assert(&pf->pf_mtx, MA_OWNED);
1874
1875         INIT_DEBUGOUT("ixl_stop: begin\n");
1876         if (pf->num_vfs == 0)
1877                 ixl_disable_intr(vsi);
1878         else
1879                 ixl_disable_rings_intr(vsi);
1880         ixl_disable_rings(vsi);
1881
1882         /* Tell the stack that the interface is no longer active */
1883         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1884
1885         /* Stop the local timer */
1886         callout_stop(&pf->timer);
1887
1888         return;
1889 }
1890
1891
1892 /*********************************************************************
1893  *
1894  *  Setup MSIX Interrupt resources and handlers for the VSI
1895  *
1896  **********************************************************************/
1897 static int
1898 ixl_assign_vsi_legacy(struct ixl_pf *pf)
1899 {
1900         device_t        dev = pf->dev;
1901         struct          ixl_vsi *vsi = &pf->vsi;
1902         struct          ixl_queue *que = vsi->queues;
1903         int             error, rid = 0;
1904
1905         if (pf->msix == 1)
1906                 rid = 1;
1907         pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1908             &rid, RF_SHAREABLE | RF_ACTIVE);
1909         if (pf->res == NULL) {
1910                 device_printf(dev,"Unable to allocate"
1911                     " bus resource: vsi legacy/msi interrupt\n");
1912                 return (ENXIO);
1913         }
1914
1915         /* Set the handler function */
1916         error = bus_setup_intr(dev, pf->res,
1917             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1918             ixl_intr, pf, &pf->tag);
1919         if (error) {
1920                 pf->res = NULL;
1921                 device_printf(dev, "Failed to register legacy/msi handler");
1922                 return (error);
1923         }
1924         bus_describe_intr(dev, pf->res, pf->tag, "irq0");
1925         TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
1926         TASK_INIT(&que->task, 0, ixl_handle_que, que);
1927         que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
1928             taskqueue_thread_enqueue, &que->tq);
1929         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1930             device_get_nameunit(dev));
1931         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1932
1933 #ifdef PCI_IOV
1934         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1935 #endif
1936
1937         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1938             taskqueue_thread_enqueue, &pf->tq);
1939         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1940             device_get_nameunit(dev));
1941
1942         return (0);
1943 }
1944
1945
1946 /*********************************************************************
1947  *
1948  *  Setup MSIX Interrupt resources and handlers for the VSI
1949  *
1950  **********************************************************************/
1951 static int
1952 ixl_assign_vsi_msix(struct ixl_pf *pf)
1953 {
1954         device_t        dev = pf->dev;
1955         struct          ixl_vsi *vsi = &pf->vsi;
1956         struct          ixl_queue *que = vsi->queues;
1957         struct          tx_ring  *txr;
1958         int             error, rid, vector = 0;
1959
1960         /* Admin Que is vector 0*/
1961         rid = vector + 1;
1962         pf->res = bus_alloc_resource_any(dev,
1963             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
1964         if (!pf->res) {
1965                 device_printf(dev,"Unable to allocate"
1966             " bus resource: Adminq interrupt [%d]\n", rid);
1967                 return (ENXIO);
1968         }
1969         /* Set the adminq vector and handler */
1970         error = bus_setup_intr(dev, pf->res,
1971             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1972             ixl_msix_adminq, pf, &pf->tag);
1973         if (error) {
1974                 pf->res = NULL;
1975                 device_printf(dev, "Failed to register Admin que handler");
1976                 return (error);
1977         }
1978         bus_describe_intr(dev, pf->res, pf->tag, "aq");
1979         pf->admvec = vector;
1980         /* Tasklet for Admin Queue */
1981         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1982
1983 #ifdef PCI_IOV
1984         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1985 #endif
1986
1987         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1988             taskqueue_thread_enqueue, &pf->tq);
1989         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1990             device_get_nameunit(pf->dev));
1991         ++vector;
1992
1993         /* Now set up the stations */
1994         for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
1995                 int cpu_id = i;
1996                 rid = vector + 1;
1997                 txr = &que->txr;
1998                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1999                     RF_SHAREABLE | RF_ACTIVE);
2000                 if (que->res == NULL) {
2001                         device_printf(dev,"Unable to allocate"
2002                             " bus resource: que interrupt [%d]\n", vector);
2003                         return (ENXIO);
2004                 }
2005                 /* Set the handler function */
2006                 error = bus_setup_intr(dev, que->res,
2007                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2008                     ixl_msix_que, que, &que->tag);
2009                 if (error) {
2010                         que->res = NULL;
2011                         device_printf(dev, "Failed to register que handler");
2012                         return (error);
2013                 }
2014                 bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2015                 /* Bind the vector to a CPU */
2016 #ifdef RSS
2017                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2018 #endif
2019                 bus_bind_intr(dev, que->res, cpu_id);
2020                 que->msix = vector;
2021                 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2022                 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2023                 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2024                     taskqueue_thread_enqueue, &que->tq);
2025 #ifdef RSS
2026                 taskqueue_start_threads_pinned(&que->tq, 1, PI_NET,
2027                     cpu_id, "%s (bucket %d)",
2028                     device_get_nameunit(dev), cpu_id);
2029 #else
2030                 taskqueue_start_threads(&que->tq, 1, PI_NET,
2031                     "%s que", device_get_nameunit(dev));
2032 #endif
2033         }
2034
2035         return (0);
2036 }
2037
2038
2039 /*
2040  * Allocate MSI/X vectors
2041  */
2042 static int
2043 ixl_init_msix(struct ixl_pf *pf)
2044 {
2045         device_t dev = pf->dev;
2046         int rid, want, vectors, queues, available;
2047
2048         /* Override by tuneable */
2049         if (ixl_enable_msix == 0)
2050                 goto msi;
2051
2052         /*
2053         ** When used in a virtualized environment
2054         ** PCI BUSMASTER capability may not be set
2055         ** so explicity set it here and rewrite
2056         ** the ENABLE in the MSIX control register
2057         ** at this point to cause the host to
2058         ** successfully initialize us.
2059         */
2060         {
2061                 u16 pci_cmd_word;
2062                 int msix_ctrl;
2063                 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2064                 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2065                 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2066                 pci_find_cap(dev, PCIY_MSIX, &rid);
2067                 rid += PCIR_MSIX_CTRL;
2068                 msix_ctrl = pci_read_config(dev, rid, 2);
2069                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2070                 pci_write_config(dev, rid, msix_ctrl, 2);
2071         }
2072
2073         /* First try MSI/X */
2074         rid = PCIR_BAR(IXL_BAR);
2075         pf->msix_mem = bus_alloc_resource_any(dev,
2076             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2077         if (!pf->msix_mem) {
2078                 /* May not be enabled */
2079                 device_printf(pf->dev,
2080                     "Unable to map MSIX table \n");
2081                 goto msi;
2082         }
2083
2084         available = pci_msix_count(dev);
2085         if (available == 0) { /* system has msix disabled */
2086                 bus_release_resource(dev, SYS_RES_MEMORY,
2087                     rid, pf->msix_mem);
2088                 pf->msix_mem = NULL;
2089                 goto msi;
2090         }
2091
2092         /* Figure out a reasonable auto config value */
2093         queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2094
2095         /* Override with hardcoded value if sane */
2096         if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2097                 queues = ixl_max_queues;
2098
2099 #ifdef  RSS
2100         /* If we're doing RSS, clamp at the number of RSS buckets */
2101         if (queues > rss_getnumbuckets())
2102                 queues = rss_getnumbuckets();
2103 #endif
2104
2105         /*
2106         ** Want one vector (RX/TX pair) per queue
2107         ** plus an additional for the admin queue.
2108         */
2109         want = queues + 1;
2110         if (want <= available)  /* Have enough */
2111                 vectors = want;
2112         else {
2113                 device_printf(pf->dev,
2114                     "MSIX Configuration Problem, "
2115                     "%d vectors available but %d wanted!\n",
2116                     available, want);
2117                 return (0); /* Will go to Legacy setup */
2118         }
2119
2120         if (pci_alloc_msix(dev, &vectors) == 0) {
2121                 device_printf(pf->dev,
2122                     "Using MSIX interrupts with %d vectors\n", vectors);
2123                 pf->msix = vectors;
2124                 pf->vsi.num_queues = queues;
2125 #ifdef RSS
2126                 /*
2127                  * If we're doing RSS, the number of queues needs to
2128                  * match the number of RSS buckets that are configured.
2129                  *
2130                  * + If there's more queues than RSS buckets, we'll end
2131                  *   up with queues that get no traffic.
2132                  *
2133                  * + If there's more RSS buckets than queues, we'll end
2134                  *   up having multiple RSS buckets map to the same queue,
2135                  *   so there'll be some contention.
2136                  */
2137                 if (queues != rss_getnumbuckets()) {
2138                         device_printf(dev,
2139                             "%s: queues (%d) != RSS buckets (%d)"
2140                             "; performance will be impacted.\n",
2141                             __func__, queues, rss_getnumbuckets());
2142                 }
2143 #endif
2144                 return (vectors);
2145         }
2146 msi:
2147         vectors = pci_msi_count(dev);
2148         pf->vsi.num_queues = 1;
2149         pf->msix = 1;
2150         ixl_max_queues = 1;
2151         ixl_enable_msix = 0;
2152         if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2153                 device_printf(pf->dev,"Using an MSI interrupt\n");
2154         else {
2155                 pf->msix = 0;
2156                 device_printf(pf->dev,"Using a Legacy interrupt\n");
2157         }
2158         return (vectors);
2159 }
2160
2161
2162 /*
2163  * Plumb MSI/X vectors
2164  */
2165 static void
2166 ixl_configure_msix(struct ixl_pf *pf)
2167 {
2168         struct i40e_hw  *hw = &pf->hw;
2169         struct ixl_vsi *vsi = &pf->vsi;
2170         u32             reg;
2171         u16             vector = 1;
2172
2173         /* First set up the adminq - vector 0 */
2174         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2175         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2176
2177         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2178             I40E_PFINT_ICR0_ENA_GRST_MASK |
2179             I40E_PFINT_ICR0_HMC_ERR_MASK |
2180             I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2181             I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2182             I40E_PFINT_ICR0_ENA_VFLR_MASK |
2183             I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2184         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2185
2186         wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2187         wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E);
2188
2189         wr32(hw, I40E_PFINT_DYN_CTL0,
2190             I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2191             I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2192
2193         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2194
2195         /* Next configure the queues */
2196         for (int i = 0; i < vsi->num_queues; i++, vector++) {
2197                 wr32(hw, I40E_PFINT_DYN_CTLN(i), i);
2198                 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2199
2200                 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2201                 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2202                 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2203                 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2204                 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2205                 wr32(hw, I40E_QINT_RQCTL(i), reg);
2206
2207                 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2208                 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2209                 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2210                 ((i+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2211                 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2212                 if (i == (vsi->num_queues - 1))
2213                         reg |= (IXL_QUEUE_EOL
2214                             << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2215                 wr32(hw, I40E_QINT_TQCTL(i), reg);
2216         }
2217 }
2218
2219 /*
2220  * Configure for MSI single vector operation
2221  */
2222 static void
2223 ixl_configure_legacy(struct ixl_pf *pf)
2224 {
2225         struct i40e_hw  *hw = &pf->hw;
2226         u32             reg;
2227
2228
2229         wr32(hw, I40E_PFINT_ITR0(0), 0);
2230         wr32(hw, I40E_PFINT_ITR0(1), 0);
2231
2232
2233         /* Setup "other" causes */
2234         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2235             | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2236             | I40E_PFINT_ICR0_ENA_GRST_MASK
2237             | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2238             | I40E_PFINT_ICR0_ENA_GPIO_MASK
2239             | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2240             | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2241             | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2242             | I40E_PFINT_ICR0_ENA_VFLR_MASK
2243             | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2244             ;
2245         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2246
2247         /* SW_ITR_IDX = 0, but don't change INTENA */
2248         wr32(hw, I40E_PFINT_DYN_CTL0,
2249             I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2250             I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2251         /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2252         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2253
2254         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2255         wr32(hw, I40E_PFINT_LNKLST0, 0);
2256
2257         /* Associate the queue pair to the vector and enable the q int */
2258         reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2259             | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2260             | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2261         wr32(hw, I40E_QINT_RQCTL(0), reg);
2262
2263         reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2264             | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2265             | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2266         wr32(hw, I40E_QINT_TQCTL(0), reg);
2267
2268         /* Next enable the queue pair */
2269         reg = rd32(hw, I40E_QTX_ENA(0));
2270         reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2271         wr32(hw, I40E_QTX_ENA(0), reg);
2272
2273         reg = rd32(hw, I40E_QRX_ENA(0));
2274         reg |= I40E_QRX_ENA_QENA_REQ_MASK;
2275         wr32(hw, I40E_QRX_ENA(0), reg);
2276 }
2277
2278
2279 /*
2280  * Set the Initial ITR state
2281  */
2282 static void
2283 ixl_configure_itr(struct ixl_pf *pf)
2284 {
2285         struct i40e_hw          *hw = &pf->hw;
2286         struct ixl_vsi          *vsi = &pf->vsi;
2287         struct ixl_queue        *que = vsi->queues;
2288
2289         vsi->rx_itr_setting = ixl_rx_itr;
2290         if (ixl_dynamic_rx_itr)
2291                 vsi->rx_itr_setting |= IXL_ITR_DYNAMIC;
2292         vsi->tx_itr_setting = ixl_tx_itr;
2293         if (ixl_dynamic_tx_itr)
2294                 vsi->tx_itr_setting |= IXL_ITR_DYNAMIC;
2295
2296         for (int i = 0; i < vsi->num_queues; i++, que++) {
2297                 struct tx_ring  *txr = &que->txr;
2298                 struct rx_ring  *rxr = &que->rxr;
2299
2300                 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2301                     vsi->rx_itr_setting);
2302                 rxr->itr = vsi->rx_itr_setting;
2303                 rxr->latency = IXL_AVE_LATENCY;
2304                 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2305                     vsi->tx_itr_setting);
2306                 txr->itr = vsi->tx_itr_setting;
2307                 txr->latency = IXL_AVE_LATENCY;
2308         }
2309 }
2310
2311
2312 static int
2313 ixl_allocate_pci_resources(struct ixl_pf *pf)
2314 {
2315         int             rid;
2316         device_t        dev = pf->dev;
2317
2318         rid = PCIR_BAR(0);
2319         pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2320             &rid, RF_ACTIVE);
2321
2322         if (!(pf->pci_mem)) {
2323                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2324                 return (ENXIO);
2325         }
2326
2327         pf->osdep.mem_bus_space_tag =
2328                 rman_get_bustag(pf->pci_mem);
2329         pf->osdep.mem_bus_space_handle =
2330                 rman_get_bushandle(pf->pci_mem);
2331         pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2332         pf->osdep.flush_reg = I40E_GLGEN_STAT;
2333         pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2334
2335         pf->hw.back = &pf->osdep;
2336
2337         /*
2338         ** Now setup MSI or MSI/X, should
2339         ** return us the number of supported
2340         ** vectors. (Will be 1 for MSI)
2341         */
2342         pf->msix = ixl_init_msix(pf);
2343         return (0);
2344 }
2345
2346 static void
2347 ixl_free_pci_resources(struct ixl_pf * pf)
2348 {
2349         struct ixl_vsi          *vsi = &pf->vsi;
2350         struct ixl_queue        *que = vsi->queues;
2351         device_t                dev = pf->dev;
2352         int                     rid, memrid;
2353
2354         memrid = PCIR_BAR(IXL_BAR);
2355
2356         /* We may get here before stations are setup */
2357         if ((!ixl_enable_msix) || (que == NULL))
2358                 goto early;
2359
2360         /*
2361         **  Release all msix VSI resources:
2362         */
2363         for (int i = 0; i < vsi->num_queues; i++, que++) {
2364                 rid = que->msix + 1;
2365                 if (que->tag != NULL) {
2366                         bus_teardown_intr(dev, que->res, que->tag);
2367                         que->tag = NULL;
2368                 }
2369                 if (que->res != NULL)
2370                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2371         }
2372
2373 early:
2374         /* Clean the AdminQ interrupt last */
2375         if (pf->admvec) /* we are doing MSIX */
2376                 rid = pf->admvec + 1;
2377         else
2378                 (pf->msix != 0) ? (rid = 1):(rid = 0);
2379
2380         if (pf->tag != NULL) {
2381                 bus_teardown_intr(dev, pf->res, pf->tag);
2382                 pf->tag = NULL;
2383         }
2384         if (pf->res != NULL)
2385                 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2386
2387         if (pf->msix)
2388                 pci_release_msi(dev);
2389
2390         if (pf->msix_mem != NULL)
2391                 bus_release_resource(dev, SYS_RES_MEMORY,
2392                     memrid, pf->msix_mem);
2393
2394         if (pf->pci_mem != NULL)
2395                 bus_release_resource(dev, SYS_RES_MEMORY,
2396                     PCIR_BAR(0), pf->pci_mem);
2397
2398         return;
2399 }
2400
2401 static void
2402 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2403 {
2404         /* Display supported media types */
2405         if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2406                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2407
2408         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2409                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2410         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2411                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2412         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2413                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2414
2415         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2416             phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4) ||
2417             phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR) ||
2418             phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2419             phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2420             phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2421             phy_type & (1 << I40E_PHY_TYPE_SFI) ||
2422             phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2423                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2424
2425         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2426                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2427         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2428                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2429         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2430                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2431
2432         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2433             phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2434             phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2435             phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2436             phy_type & (1 << I40E_PHY_TYPE_XLPPI) ||
2437             /* KR4 uses CR4 until the OS has the real media type */
2438             phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2439                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2440
2441         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2442                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2443         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2444                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2445 }
2446
2447 /*********************************************************************
2448  *
2449  *  Setup networking device structure and register an interface.
2450  *
2451  **********************************************************************/
2452 static int
2453 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2454 {
2455         struct ifnet            *ifp;
2456         struct i40e_hw          *hw = vsi->hw;
2457         struct ixl_queue        *que = vsi->queues;
2458         struct i40e_aq_get_phy_abilities_resp abilities;
2459         enum i40e_status_code aq_error = 0;
2460
2461         INIT_DEBUGOUT("ixl_setup_interface: begin");
2462
2463         ifp = vsi->ifp = if_alloc(IFT_ETHER);
2464         if (ifp == NULL) {
2465                 device_printf(dev, "can not allocate ifnet structure\n");
2466                 return (-1);
2467         }
2468         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2469         ifp->if_mtu = ETHERMTU;
2470         if_initbaudrate(ifp, IF_Gbps(40));
2471         ifp->if_init = ixl_init;
2472         ifp->if_softc = vsi;
2473         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2474         ifp->if_ioctl = ixl_ioctl;
2475
2476 #if __FreeBSD_version >= 1100036
2477         if_setgetcounterfn(ifp, ixl_get_counter);
2478 #endif
2479
2480         ifp->if_transmit = ixl_mq_start;
2481
2482         ifp->if_qflush = ixl_qflush;
2483
2484         ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2485
2486         vsi->max_frame_size =
2487             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2488             + ETHER_VLAN_ENCAP_LEN;
2489
2490         /*
2491          * Tell the upper layer(s) we support long frames.
2492          */
2493         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2494
2495         ifp->if_capabilities |= IFCAP_HWCSUM;
2496         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2497         ifp->if_capabilities |= IFCAP_TSO;
2498         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2499         ifp->if_capabilities |= IFCAP_LRO;
2500
2501         /* VLAN capabilties */
2502         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2503                              |  IFCAP_VLAN_HWTSO
2504                              |  IFCAP_VLAN_MTU
2505                              |  IFCAP_VLAN_HWCSUM;
2506         ifp->if_capenable = ifp->if_capabilities;
2507
2508         /*
2509         ** Don't turn this on by default, if vlans are
2510         ** created on another pseudo device (eg. lagg)
2511         ** then vlan events are not passed thru, breaking
2512         ** operation, but with HW FILTER off it works. If
2513         ** using vlans directly on the ixl driver you can
2514         ** enable this and get full hardware tag filtering.
2515         */
2516         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2517
2518         /*
2519          * Specify the media types supported by this adapter and register
2520          * callbacks to update media and link information
2521          */
2522         ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2523                      ixl_media_status);
2524
2525         aq_error = i40e_aq_get_phy_capabilities(hw,
2526             FALSE, TRUE, &abilities, NULL);
2527         /* May need delay to detect fiber correctly */
2528         if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2529                 i40e_msec_delay(200);
2530                 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2531                     TRUE, &abilities, NULL);
2532         }
2533         if (aq_error) {
2534                 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2535                         device_printf(dev, "Unknown PHY type detected!\n");
2536                 else
2537                         device_printf(dev,
2538                             "Error getting supported media types, err %d,"
2539                             " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2540                 return (0);
2541         }
2542
2543         ixl_add_ifmedia(vsi, abilities.phy_type);
2544
2545         /* Use autoselect media by default */
2546         ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2547         ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2548
2549         ether_ifattach(ifp, hw->mac.addr);
2550
2551         return (0);
2552 }
2553
2554 /*
2555 ** Run when the Admin Queue gets a
2556 ** link transition interrupt.
2557 */
2558 static void
2559 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
2560 {
2561         struct i40e_hw  *hw = &pf->hw;
2562         struct i40e_aqc_get_link_status *status =
2563             (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
2564         bool check;
2565
2566         hw->phy.get_link_info = TRUE;
2567         check = i40e_get_link_status(hw);
2568         pf->link_up = check;
2569 #ifdef IXL_DEBUG
2570         printf("Link is %s\n", check ? "up":"down");
2571 #endif
2572         /* Report if Unqualified modules are found */
2573         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
2574             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
2575             (!(status->link_info & I40E_AQ_LINK_UP)))
2576                 device_printf(pf->dev, "Link failed because "
2577                     "an unqualified module was detected\n");
2578
2579         return;
2580 }
2581
2582 /*********************************************************************
2583  *
2584  *  Get Firmware Switch configuration
2585  *      - this will need to be more robust when more complex
2586  *        switch configurations are enabled.
2587  *
2588  **********************************************************************/
2589 static int
2590 ixl_switch_config(struct ixl_pf *pf)
2591 {
2592         struct i40e_hw  *hw = &pf->hw;
2593         struct ixl_vsi  *vsi = &pf->vsi;
2594         device_t        dev = vsi->dev;
2595         struct i40e_aqc_get_switch_config_resp *sw_config;
2596         u8      aq_buf[I40E_AQ_LARGE_BUF];
2597         int     ret;
2598         u16     next = 0;
2599
2600         memset(&aq_buf, 0, sizeof(aq_buf));
2601         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
2602         ret = i40e_aq_get_switch_config(hw, sw_config,
2603             sizeof(aq_buf), &next, NULL);
2604         if (ret) {
2605                 device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n",
2606                     ret);
2607                 return (ret);
2608         }
2609 #ifdef IXL_DEBUG
2610         device_printf(dev,
2611             "Switch config: header reported: %d in structure, %d total\n",
2612             sw_config->header.num_reported, sw_config->header.num_total);
2613         for (int i = 0; i < sw_config->header.num_reported; i++) {
2614                 device_printf(dev,
2615                     "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
2616                     sw_config->element[i].element_type,
2617                     sw_config->element[i].seid,
2618                     sw_config->element[i].uplink_seid,
2619                     sw_config->element[i].downlink_seid);
2620         }
2621 #endif
2622         /* Simplified due to a single VSI at the moment */
2623         vsi->uplink_seid = sw_config->element[0].uplink_seid;
2624         vsi->downlink_seid = sw_config->element[0].downlink_seid;
2625         vsi->seid = sw_config->element[0].seid;
2626         return (ret);
2627 }
2628
2629 /*********************************************************************
2630  *
2631  *  Initialize the VSI:  this handles contexts, which means things
2632  *                       like the number of descriptors, buffer size,
2633  *                       plus we init the rings thru this function.
2634  *
2635  **********************************************************************/
2636 static int
2637 ixl_initialize_vsi(struct ixl_vsi *vsi)
2638 {
2639         struct ixl_pf           *pf = vsi->back;
2640         struct ixl_queue        *que = vsi->queues;
2641         device_t                dev = vsi->dev;
2642         struct i40e_hw          *hw = vsi->hw;
2643         struct i40e_vsi_context ctxt;
2644         int                     err = 0;
2645
2646         memset(&ctxt, 0, sizeof(ctxt));
2647         ctxt.seid = vsi->seid;
2648         if (pf->veb_seid != 0)
2649                 ctxt.uplink_seid = pf->veb_seid;
2650         ctxt.pf_num = hw->pf_id;
2651         err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2652         if (err) {
2653                 device_printf(dev,"get vsi params failed %x!!\n", err);
2654                 return (err);
2655         }
2656 #ifdef IXL_DEBUG
2657         printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
2658             "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
2659             "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
2660             ctxt.uplink_seid, ctxt.vsi_number,
2661             ctxt.vsis_allocated, ctxt.vsis_unallocated,
2662             ctxt.flags, ctxt.pf_num, ctxt.vf_num,
2663             ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
2664 #endif
2665         /*
2666         ** Set the queue and traffic class bits
2667         **  - when multiple traffic classes are supported
2668         **    this will need to be more robust.
2669         */
2670         ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2671         ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
2672         ctxt.info.queue_mapping[0] = 0;
2673         ctxt.info.tc_mapping[0] = 0x0800;
2674
2675         /* Set VLAN receive stripping mode */
2676         ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
2677         ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
2678         if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
2679             ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2680         else
2681             ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2682
2683         /* Keep copy of VSI info in VSI for statistic counters */
2684         memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2685
2686         /* Reset VSI statistics */
2687         ixl_vsi_reset_stats(vsi);
2688         vsi->hw_filters_add = 0;
2689         vsi->hw_filters_del = 0;
2690
2691         ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
2692
2693         err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2694         if (err) {
2695                 device_printf(dev,"update vsi params failed %x!!\n",
2696                    hw->aq.asq_last_status);
2697                 return (err);
2698         }
2699
2700         for (int i = 0; i < vsi->num_queues; i++, que++) {
2701                 struct tx_ring          *txr = &que->txr;
2702                 struct rx_ring          *rxr = &que->rxr;
2703                 struct i40e_hmc_obj_txq tctx;
2704                 struct i40e_hmc_obj_rxq rctx;
2705                 u32                     txctl;
2706                 u16                     size;
2707
2708
2709                 /* Setup the HMC TX Context  */
2710                 size = que->num_desc * sizeof(struct i40e_tx_desc);
2711                 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
2712                 tctx.new_context = 1;
2713                 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
2714                 tctx.qlen = que->num_desc;
2715                 tctx.fc_ena = 0;
2716                 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
2717                 /* Enable HEAD writeback */
2718                 tctx.head_wb_ena = 1;
2719                 tctx.head_wb_addr = txr->dma.pa +
2720                     (que->num_desc * sizeof(struct i40e_tx_desc));
2721                 tctx.rdylist_act = 0;
2722                 err = i40e_clear_lan_tx_queue_context(hw, i);
2723                 if (err) {
2724                         device_printf(dev, "Unable to clear TX context\n");
2725                         break;
2726                 }
2727                 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
2728                 if (err) {
2729                         device_printf(dev, "Unable to set TX context\n");
2730                         break;
2731                 }
2732                 /* Associate the ring with this PF */
2733                 txctl = I40E_QTX_CTL_PF_QUEUE;
2734                 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2735                     I40E_QTX_CTL_PF_INDX_MASK);
2736                 wr32(hw, I40E_QTX_CTL(i), txctl);
2737                 ixl_flush(hw);
2738
2739                 /* Do ring (re)init */
2740                 ixl_init_tx_ring(que);
2741
2742                 /* Next setup the HMC RX Context  */
2743                 if (vsi->max_frame_size <= MCLBYTES)
2744                         rxr->mbuf_sz = MCLBYTES;
2745                 else
2746                         rxr->mbuf_sz = MJUMPAGESIZE;
2747
2748                 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
2749
2750                 /* Set up an RX context for the HMC */
2751                 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2752                 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
2753                 /* ignore header split for now */
2754                 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
2755                 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
2756                     vsi->max_frame_size : max_rxmax;
2757                 rctx.dtype = 0;
2758                 rctx.dsize = 1; /* do 32byte descriptors */
2759                 rctx.hsplit_0 = 0;  /* no HDR split initially */
2760                 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
2761                 rctx.qlen = que->num_desc;
2762                 rctx.tphrdesc_ena = 1;
2763                 rctx.tphwdesc_ena = 1;
2764                 rctx.tphdata_ena = 0;
2765                 rctx.tphhead_ena = 0;
2766                 rctx.lrxqthresh = 2;
2767                 rctx.crcstrip = 1;
2768                 rctx.l2tsel = 1;
2769                 rctx.showiv = 1;
2770                 rctx.fc_ena = 0;
2771                 rctx.prefena = 1;
2772
2773                 err = i40e_clear_lan_rx_queue_context(hw, i);
2774                 if (err) {
2775                         device_printf(dev,
2776                             "Unable to clear RX context %d\n", i);
2777                         break;
2778                 }
2779                 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
2780                 if (err) {
2781                         device_printf(dev, "Unable to set RX context %d\n", i);
2782                         break;
2783                 }
2784                 err = ixl_init_rx_ring(que);
2785                 if (err) {
2786                         device_printf(dev, "Fail in init_rx_ring %d\n", i);
2787                         break;
2788                 }
2789                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), 0);
2790 #ifdef DEV_NETMAP
2791                 /* preserve queue */
2792                 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
2793                         struct netmap_adapter *na = NA(vsi->ifp);
2794                         struct netmap_kring *kring = &na->rx_rings[i];
2795                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
2796                         wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
2797                 } else
2798 #endif /* DEV_NETMAP */
2799                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
2800         }
2801         return (err);
2802 }
2803
2804
2805 /*********************************************************************
2806  *
2807  *  Free all VSI structs.
2808  *
2809  **********************************************************************/
2810 void
2811 ixl_free_vsi(struct ixl_vsi *vsi)
2812 {
2813         struct ixl_pf           *pf = (struct ixl_pf *)vsi->back;
2814         struct ixl_queue        *que = vsi->queues;
2815
2816         /* Free station queues */
2817         for (int i = 0; i < vsi->num_queues; i++, que++) {
2818                 struct tx_ring *txr = &que->txr;
2819                 struct rx_ring *rxr = &que->rxr;
2820
2821                 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
2822                         continue;
2823                 IXL_TX_LOCK(txr);
2824                 ixl_free_que_tx(que);
2825                 if (txr->base)
2826                         i40e_free_dma_mem(&pf->hw, &txr->dma);
2827                 IXL_TX_UNLOCK(txr);
2828                 IXL_TX_LOCK_DESTROY(txr);
2829
2830                 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
2831                         continue;
2832                 IXL_RX_LOCK(rxr);
2833                 ixl_free_que_rx(que);
2834                 if (rxr->base)
2835                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
2836                 IXL_RX_UNLOCK(rxr);
2837                 IXL_RX_LOCK_DESTROY(rxr);
2838
2839         }
2840         free(vsi->queues, M_DEVBUF);
2841
2842         /* Free VSI filter list */
2843         ixl_free_mac_filters(vsi);
2844 }
2845
2846 static void
2847 ixl_free_mac_filters(struct ixl_vsi *vsi)
2848 {
2849         struct ixl_mac_filter *f;
2850
2851         while (!SLIST_EMPTY(&vsi->ftl)) {
2852                 f = SLIST_FIRST(&vsi->ftl);
2853                 SLIST_REMOVE_HEAD(&vsi->ftl, next);
2854                 free(f, M_DEVBUF);
2855         }
2856 }
2857
2858
2859 /*********************************************************************
2860  *
2861  *  Allocate memory for the VSI (virtual station interface) and their
2862  *  associated queues, rings and the descriptors associated with each,
2863  *  called only once at attach.
2864  *
2865  **********************************************************************/
2866 static int
2867 ixl_setup_stations(struct ixl_pf *pf)
2868 {
2869         device_t                dev = pf->dev;
2870         struct ixl_vsi          *vsi;
2871         struct ixl_queue        *que;
2872         struct tx_ring          *txr;
2873         struct rx_ring          *rxr;
2874         int                     rsize, tsize;
2875         int                     error = I40E_SUCCESS;
2876
2877         vsi = &pf->vsi;
2878         vsi->back = (void *)pf;
2879         vsi->hw = &pf->hw;
2880         vsi->id = 0;
2881         vsi->num_vlans = 0;
2882         vsi->back = pf;
2883
2884         /* Get memory for the station queues */
2885         if (!(vsi->queues =
2886             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
2887             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2888                 device_printf(dev, "Unable to allocate queue memory\n");
2889                 error = ENOMEM;
2890                 goto early;
2891         }
2892
2893         for (int i = 0; i < vsi->num_queues; i++) {
2894                 que = &vsi->queues[i];
2895                 que->num_desc = ixl_ringsz;
2896                 que->me = i;
2897                 que->vsi = vsi;
2898                 /* mark the queue as active */
2899                 vsi->active_queues |= (u64)1 << que->me;
2900                 txr = &que->txr;
2901                 txr->que = que;
2902                 txr->tail = I40E_QTX_TAIL(que->me);
2903
2904                 /* Initialize the TX lock */
2905                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2906                     device_get_nameunit(dev), que->me);
2907                 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
2908                 /* Create the TX descriptor ring */
2909                 tsize = roundup2((que->num_desc *
2910                     sizeof(struct i40e_tx_desc)) +
2911                     sizeof(u32), DBA_ALIGN);
2912                 if (i40e_allocate_dma_mem(&pf->hw,
2913                     &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
2914                         device_printf(dev,
2915                             "Unable to allocate TX Descriptor memory\n");
2916                         error = ENOMEM;
2917                         goto fail;
2918                 }
2919                 txr->base = (struct i40e_tx_desc *)txr->dma.va;
2920                 bzero((void *)txr->base, tsize);
2921                 /* Now allocate transmit soft structs for the ring */
2922                 if (ixl_allocate_tx_data(que)) {
2923                         device_printf(dev,
2924                             "Critical Failure setting up TX structures\n");
2925                         error = ENOMEM;
2926                         goto fail;
2927                 }
2928                 /* Allocate a buf ring */
2929                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
2930                     M_WAITOK, &txr->mtx);
2931                 if (txr->br == NULL) {
2932                         device_printf(dev,
2933                             "Critical Failure setting up TX buf ring\n");
2934                         error = ENOMEM;
2935                         goto fail;
2936                 }
2937
2938                 /*
2939                  * Next the RX queues...
2940                  */
2941                 rsize = roundup2(que->num_desc *
2942                     sizeof(union i40e_rx_desc), DBA_ALIGN);
2943                 rxr = &que->rxr;
2944                 rxr->que = que;
2945                 rxr->tail = I40E_QRX_TAIL(que->me);
2946
2947                 /* Initialize the RX side lock */
2948                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2949                     device_get_nameunit(dev), que->me);
2950                 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
2951
2952                 if (i40e_allocate_dma_mem(&pf->hw,
2953                     &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
2954                         device_printf(dev,
2955                             "Unable to allocate RX Descriptor memory\n");
2956                         error = ENOMEM;
2957                         goto fail;
2958                 }
2959                 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
2960                 bzero((void *)rxr->base, rsize);
2961
2962                 /* Allocate receive soft structs for the ring*/
2963                 if (ixl_allocate_rx_data(que)) {
2964                         device_printf(dev,
2965                             "Critical Failure setting up receive structs\n");
2966                         error = ENOMEM;
2967                         goto fail;
2968                 }
2969         }
2970
2971         return (0);
2972
2973 fail:
2974         for (int i = 0; i < vsi->num_queues; i++) {
2975                 que = &vsi->queues[i];
2976                 rxr = &que->rxr;
2977                 txr = &que->txr;
2978                 if (rxr->base)
2979                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
2980                 if (txr->base)
2981                         i40e_free_dma_mem(&pf->hw, &txr->dma);
2982         }
2983
2984 early:
2985         return (error);
2986 }
2987
2988 /*
2989 ** Provide a update to the queue RX
2990 ** interrupt moderation value.
2991 */
2992 static void
2993 ixl_set_queue_rx_itr(struct ixl_queue *que)
2994 {
2995         struct ixl_vsi  *vsi = que->vsi;
2996         struct i40e_hw  *hw = vsi->hw;
2997         struct rx_ring  *rxr = &que->rxr;
2998         u16             rx_itr;
2999         u16             rx_latency = 0;
3000         int             rx_bytes;
3001
3002
3003         /* Idle, do nothing */
3004         if (rxr->bytes == 0)
3005                 return;
3006
3007         if (ixl_dynamic_rx_itr) {
3008                 rx_bytes = rxr->bytes/rxr->itr;
3009                 rx_itr = rxr->itr;
3010
3011                 /* Adjust latency range */
3012                 switch (rxr->latency) {
3013                 case IXL_LOW_LATENCY:
3014                         if (rx_bytes > 10) {
3015                                 rx_latency = IXL_AVE_LATENCY;
3016                                 rx_itr = IXL_ITR_20K;
3017                         }
3018                         break;
3019                 case IXL_AVE_LATENCY:
3020                         if (rx_bytes > 20) {
3021                                 rx_latency = IXL_BULK_LATENCY;
3022                                 rx_itr = IXL_ITR_8K;
3023                         } else if (rx_bytes <= 10) {
3024                                 rx_latency = IXL_LOW_LATENCY;
3025                                 rx_itr = IXL_ITR_100K;
3026                         }
3027                         break;
3028                 case IXL_BULK_LATENCY:
3029                         if (rx_bytes <= 20) {
3030                                 rx_latency = IXL_AVE_LATENCY;
3031                                 rx_itr = IXL_ITR_20K;
3032                         }
3033                         break;
3034                  }
3035
3036                 rxr->latency = rx_latency;
3037
3038                 if (rx_itr != rxr->itr) {
3039                         /* do an exponential smoothing */
3040                         rx_itr = (10 * rx_itr * rxr->itr) /
3041                             ((9 * rx_itr) + rxr->itr);
3042                         rxr->itr = rx_itr & IXL_MAX_ITR;
3043                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3044                             que->me), rxr->itr);
3045                 }
3046         } else { /* We may have have toggled to non-dynamic */
3047                 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3048                         vsi->rx_itr_setting = ixl_rx_itr;
3049                 /* Update the hardware if needed */
3050                 if (rxr->itr != vsi->rx_itr_setting) {
3051                         rxr->itr = vsi->rx_itr_setting;
3052                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3053                             que->me), rxr->itr);
3054                 }
3055         }
3056         rxr->bytes = 0;
3057         rxr->packets = 0;
3058         return;
3059 }
3060
3061
3062 /*
3063 ** Provide a update to the queue TX
3064 ** interrupt moderation value.
3065 */
3066 static void
3067 ixl_set_queue_tx_itr(struct ixl_queue *que)
3068 {
3069         struct ixl_vsi  *vsi = que->vsi;
3070         struct i40e_hw  *hw = vsi->hw;
3071         struct tx_ring  *txr = &que->txr;
3072         u16             tx_itr;
3073         u16             tx_latency = 0;
3074         int             tx_bytes;
3075
3076
3077         /* Idle, do nothing */
3078         if (txr->bytes == 0)
3079                 return;
3080
3081         if (ixl_dynamic_tx_itr) {
3082                 tx_bytes = txr->bytes/txr->itr;
3083                 tx_itr = txr->itr;
3084
3085                 switch (txr->latency) {
3086                 case IXL_LOW_LATENCY:
3087                         if (tx_bytes > 10) {
3088                                 tx_latency = IXL_AVE_LATENCY;
3089                                 tx_itr = IXL_ITR_20K;
3090                         }
3091                         break;
3092                 case IXL_AVE_LATENCY:
3093                         if (tx_bytes > 20) {
3094                                 tx_latency = IXL_BULK_LATENCY;
3095                                 tx_itr = IXL_ITR_8K;
3096                         } else if (tx_bytes <= 10) {
3097                                 tx_latency = IXL_LOW_LATENCY;
3098                                 tx_itr = IXL_ITR_100K;
3099                         }
3100                         break;
3101                 case IXL_BULK_LATENCY:
3102                         if (tx_bytes <= 20) {
3103                                 tx_latency = IXL_AVE_LATENCY;
3104                                 tx_itr = IXL_ITR_20K;
3105                         }
3106                         break;
3107                 }
3108
3109                 txr->latency = tx_latency;
3110
3111                 if (tx_itr != txr->itr) {
3112                  /* do an exponential smoothing */
3113                         tx_itr = (10 * tx_itr * txr->itr) /
3114                             ((9 * tx_itr) + txr->itr);
3115                         txr->itr = tx_itr & IXL_MAX_ITR;
3116                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3117                             que->me), txr->itr);
3118                 }
3119
3120         } else { /* We may have have toggled to non-dynamic */
3121                 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3122                         vsi->tx_itr_setting = ixl_tx_itr;
3123                 /* Update the hardware if needed */
3124                 if (txr->itr != vsi->tx_itr_setting) {
3125                         txr->itr = vsi->tx_itr_setting;
3126                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3127                             que->me), txr->itr);
3128                 }
3129         }
3130         txr->bytes = 0;
3131         txr->packets = 0;
3132         return;
3133 }
3134
3135 #define QUEUE_NAME_LEN 32
3136
3137 static void
3138 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3139     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3140 {
3141         struct sysctl_oid *tree;
3142         struct sysctl_oid_list *child;
3143         struct sysctl_oid_list *vsi_list;
3144
3145         tree = device_get_sysctl_tree(pf->dev);
3146         child = SYSCTL_CHILDREN(tree);
3147         vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3148                                    CTLFLAG_RD, NULL, "VSI Number");
3149         vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3150
3151         ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3152 }
3153
3154 static void
3155 ixl_add_hw_stats(struct ixl_pf *pf)
3156 {
3157         device_t dev = pf->dev;
3158         struct ixl_vsi *vsi = &pf->vsi;
3159         struct ixl_queue *queues = vsi->queues;
3160         struct i40e_hw_port_stats *pf_stats = &pf->stats;
3161
3162         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3163         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3164         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3165         struct sysctl_oid_list *vsi_list;
3166
3167         struct sysctl_oid *queue_node;
3168         struct sysctl_oid_list *queue_list;
3169
3170         struct tx_ring *txr;
3171         struct rx_ring *rxr;
3172         char queue_namebuf[QUEUE_NAME_LEN];
3173
3174         /* Driver statistics */
3175         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3176                         CTLFLAG_RD, &pf->watchdog_events,
3177                         "Watchdog timeouts");
3178         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3179                         CTLFLAG_RD, &pf->admin_irq,
3180                         "Admin Queue IRQ Handled");
3181
3182         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "vc_debug_level",
3183             CTLFLAG_RW, &pf->vc_debug_lvl, 0,
3184             "PF/VF Virtual Channel debug logging level");
3185
3186         ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3187         vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3188
3189         /* Queue statistics */
3190         for (int q = 0; q < vsi->num_queues; q++) {
3191                 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3192                 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3193                     OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3194                 queue_list = SYSCTL_CHILDREN(queue_node);
3195
3196                 txr = &(queues[q].txr);
3197                 rxr = &(queues[q].rxr);
3198
3199                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3200                                 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3201                                 "m_defrag() failed");
3202                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped",
3203                                 CTLFLAG_RD, &(queues[q].dropped_pkts),
3204                                 "Driver dropped packets");
3205                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3206                                 CTLFLAG_RD, &(queues[q].irqs),
3207                                 "irqs on this queue");
3208                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3209                                 CTLFLAG_RD, &(queues[q].tso),
3210                                 "TSO");
3211                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3212                                 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3213                                 "Driver tx dma failure in xmit");
3214                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3215                                 CTLFLAG_RD, &(txr->no_desc),
3216                                 "Queue No Descriptor Available");
3217                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3218                                 CTLFLAG_RD, &(txr->total_packets),
3219                                 "Queue Packets Transmitted");
3220                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3221                                 CTLFLAG_RD, &(txr->tx_bytes),
3222                                 "Queue Bytes Transmitted");
3223                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3224                                 CTLFLAG_RD, &(rxr->rx_packets),
3225                                 "Queue Packets Received");
3226                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3227                                 CTLFLAG_RD, &(rxr->rx_bytes),
3228                                 "Queue Bytes Received");
3229         }
3230
3231         /* MAC stats */
3232         ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3233 }
3234
3235 static void
3236 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3237         struct sysctl_oid_list *child,
3238         struct i40e_eth_stats *eth_stats)
3239 {
3240         struct ixl_sysctl_info ctls[] =
3241         {
3242                 {&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3243                 {&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3244                         "Unicast Packets Received"},
3245                 {&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3246                         "Multicast Packets Received"},
3247                 {&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3248                         "Broadcast Packets Received"},
3249                 {&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3250                 {&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3251                 {&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3252                 {&eth_stats->tx_multicast, "mcast_pkts_txd",
3253                         "Multicast Packets Transmitted"},
3254                 {&eth_stats->tx_broadcast, "bcast_pkts_txd",
3255                         "Broadcast Packets Transmitted"},
3256                 // end
3257                 {0,0,0}
3258         };
3259
3260         struct ixl_sysctl_info *entry = ctls;
3261         while (entry->stat != 0)
3262         {
3263                 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3264                                 CTLFLAG_RD, entry->stat,
3265                                 entry->description);
3266                 entry++;
3267         }
3268 }
3269
3270 static void
3271 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3272         struct sysctl_oid_list *child,
3273         struct i40e_hw_port_stats *stats)
3274 {
3275         struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3276                                     CTLFLAG_RD, NULL, "Mac Statistics");
3277         struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3278
3279         struct i40e_eth_stats *eth_stats = &stats->eth;
3280         ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3281
3282         struct ixl_sysctl_info ctls[] =
3283         {
3284                 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3285                 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3286                 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3287                 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3288                 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3289                 /* Packet Reception Stats */
3290                 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3291                 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3292                 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3293                 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3294                 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3295                 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3296                 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3297                 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3298                 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3299                 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3300                 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3301                 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3302                 /* Packet Transmission Stats */
3303                 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3304                 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3305                 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3306                 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3307                 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3308                 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3309                 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3310                 /* Flow control */
3311                 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3312                 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3313                 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3314                 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3315                 /* End */
3316                 {0,0,0}
3317         };
3318
3319         struct ixl_sysctl_info *entry = ctls;
3320         while (entry->stat != 0)
3321         {
3322                 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3323                                 CTLFLAG_RD, entry->stat,
3324                                 entry->description);
3325                 entry++;
3326         }
3327 }
3328
3329 /*
3330 ** ixl_config_rss - setup RSS
3331 **  - note this is done for the single vsi
3332 */
3333 static void ixl_config_rss(struct ixl_vsi *vsi)
3334 {
3335         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3336         struct i40e_hw  *hw = vsi->hw;
3337         u32             lut = 0;
3338         u64             set_hena = 0, hena;
3339         int             i, j, que_id;
3340 #ifdef RSS
3341         u32             rss_hash_config;
3342         u32             rss_seed[IXL_KEYSZ];
3343 #else
3344         u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3345                             0x183cfd8c, 0xce880440, 0x580cbc3c,
3346                             0x35897377, 0x328b25e1, 0x4fa98922,
3347                             0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3348 #endif
3349
3350 #ifdef RSS
3351         /* Fetch the configured RSS key */
3352         rss_getkey((uint8_t *) &rss_seed);
3353 #endif
3354
3355         /* Fill out hash function seed */
3356         for (i = 0; i < IXL_KEYSZ; i++)
3357                 wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3358
3359         /* Enable PCTYPES for RSS: */
3360 #ifdef RSS
3361         rss_hash_config = rss_gethashconfig();
3362         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3363                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3364         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3365                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3366         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3367                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3368         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3369                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3370         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3371                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3372         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3373                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3374         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3375                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3376 #else
3377         set_hena =
3378                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3379                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3380                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3381                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3382                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3383                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3384                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3385                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3386                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3387                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3388                 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3389 #endif
3390         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
3391             ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
3392         hena |= set_hena;
3393         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
3394         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3395
3396         /* Populate the LUT with max no. of queues in round robin fashion */
3397         for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3398                 if (j == vsi->num_queues)
3399                         j = 0;
3400 #ifdef RSS
3401                 /*
3402                  * Fetch the RSS bucket id for the given indirection entry.
3403                  * Cap it at the number of configured buckets (which is
3404                  * num_queues.)
3405                  */
3406                 que_id = rss_get_indirection_to_bucket(i);
3407                 que_id = que_id % vsi->num_queues;
3408 #else
3409                 que_id = j;
3410 #endif
3411                 /* lut = 4-byte sliding window of 4 lut entries */
3412                 lut = (lut << 8) | (que_id &
3413                     ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3414                 /* On i = 3, we have 4 entries in lut; write to the register */
3415                 if ((i & 3) == 3)
3416                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3417         }
3418         ixl_flush(hw);
3419 }
3420
3421
3422 /*
3423 ** This routine is run via an vlan config EVENT,
3424 ** it enables us to use the HW Filter table since
3425 ** we can get the vlan id. This just creates the
3426 ** entry in the soft version of the VFTA, init will
3427 ** repopulate the real table.
3428 */
3429 static void
3430 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3431 {
3432         struct ixl_vsi  *vsi = ifp->if_softc;
3433         struct i40e_hw  *hw = vsi->hw;
3434         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3435
3436         if (ifp->if_softc !=  arg)   /* Not our event */
3437                 return;
3438
3439         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3440                 return;
3441
3442         IXL_PF_LOCK(pf);
3443         ++vsi->num_vlans;
3444         ixl_add_filter(vsi, hw->mac.addr, vtag);
3445         IXL_PF_UNLOCK(pf);
3446 }
3447
3448 /*
3449 ** This routine is run via an vlan
3450 ** unconfig EVENT, remove our entry
3451 ** in the soft vfta.
3452 */
3453 static void
3454 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3455 {
3456         struct ixl_vsi  *vsi = ifp->if_softc;
3457         struct i40e_hw  *hw = vsi->hw;
3458         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3459
3460         if (ifp->if_softc !=  arg)
3461                 return;
3462
3463         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3464                 return;
3465
3466         IXL_PF_LOCK(pf);
3467         --vsi->num_vlans;
3468         ixl_del_filter(vsi, hw->mac.addr, vtag);
3469         IXL_PF_UNLOCK(pf);
3470 }
3471
3472 /*
3473 ** This routine updates vlan filters, called by init
3474 ** it scans the filter table and then updates the hw
3475 ** after a soft reset.
3476 */
3477 static void
3478 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
3479 {
3480         struct ixl_mac_filter   *f;
3481         int                     cnt = 0, flags;
3482
3483         if (vsi->num_vlans == 0)
3484                 return;
3485         /*
3486         ** Scan the filter list for vlan entries,
3487         ** mark them for addition and then call
3488         ** for the AQ update.
3489         */
3490         SLIST_FOREACH(f, &vsi->ftl, next) {
3491                 if (f->flags & IXL_FILTER_VLAN) {
3492                         f->flags |=
3493                             (IXL_FILTER_ADD |
3494                             IXL_FILTER_USED);
3495                         cnt++;
3496                 }
3497         }
3498         if (cnt == 0) {
3499                 printf("setup vlan: no filters found!\n");
3500                 return;
3501         }
3502         flags = IXL_FILTER_VLAN;
3503         flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3504         ixl_add_hw_filters(vsi, flags, cnt);
3505         return;
3506 }
3507
3508 /*
3509 ** Initialize filter list and add filters that the hardware
3510 ** needs to know about.
3511 */
3512 static void
3513 ixl_init_filters(struct ixl_vsi *vsi)
3514 {
3515         /* Add broadcast address */
3516         ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
3517 }
3518
3519 /*
3520 ** This routine adds mulicast filters
3521 */
3522 static void
3523 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
3524 {
3525         struct ixl_mac_filter *f;
3526
3527         /* Does one already exist */
3528         f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3529         if (f != NULL)
3530                 return;
3531
3532         f = ixl_get_filter(vsi);
3533         if (f == NULL) {
3534                 printf("WARNING: no filter available!!\n");
3535                 return;
3536         }
3537         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3538         f->vlan = IXL_VLAN_ANY;
3539         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
3540             | IXL_FILTER_MC);
3541
3542         return;
3543 }
3544
3545 static void
3546 ixl_reconfigure_filters(struct ixl_vsi *vsi)
3547 {
3548
3549         ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
3550 }
3551
3552 /*
3553 ** This routine adds macvlan filters
3554 */
3555 static void
3556 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3557 {
3558         struct ixl_mac_filter   *f, *tmp;
3559         struct ixl_pf           *pf;
3560         device_t                dev;
3561
3562         DEBUGOUT("ixl_add_filter: begin");
3563
3564         pf = vsi->back;
3565         dev = pf->dev;
3566
3567         /* Does one already exist */
3568         f = ixl_find_filter(vsi, macaddr, vlan);
3569         if (f != NULL)
3570                 return;
3571         /*
3572         ** Is this the first vlan being registered, if so we
3573         ** need to remove the ANY filter that indicates we are
3574         ** not in a vlan, and replace that with a 0 filter.
3575         */
3576         if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
3577                 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3578                 if (tmp != NULL) {
3579                         ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
3580                         ixl_add_filter(vsi, macaddr, 0);
3581                 }
3582         }
3583
3584         f = ixl_get_filter(vsi);
3585         if (f == NULL) {
3586                 device_printf(dev, "WARNING: no filter available!!\n");
3587                 return;
3588         }
3589         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3590         f->vlan = vlan;
3591         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3592         if (f->vlan != IXL_VLAN_ANY)
3593                 f->flags |= IXL_FILTER_VLAN;
3594         else
3595                 vsi->num_macs++;
3596
3597         ixl_add_hw_filters(vsi, f->flags, 1);
3598         return;
3599 }
3600
3601 static void
3602 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3603 {
3604         struct ixl_mac_filter *f;
3605
3606         f = ixl_find_filter(vsi, macaddr, vlan);
3607         if (f == NULL)
3608                 return;
3609
3610         f->flags |= IXL_FILTER_DEL;
3611         ixl_del_hw_filters(vsi, 1);
3612         vsi->num_macs--;
3613
3614         /* Check if this is the last vlan removal */
3615         if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
3616                 /* Switch back to a non-vlan filter */
3617                 ixl_del_filter(vsi, macaddr, 0);
3618                 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
3619         }
3620         return;
3621 }
3622
3623 /*
3624 ** Find the filter with both matching mac addr and vlan id
3625 */
3626 static struct ixl_mac_filter *
3627 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3628 {
3629         struct ixl_mac_filter   *f;
3630         bool                    match = FALSE;
3631
3632         SLIST_FOREACH(f, &vsi->ftl, next) {
3633                 if (!cmp_etheraddr(f->macaddr, macaddr))
3634                         continue;
3635                 if (f->vlan == vlan) {
3636                         match = TRUE;
3637                         break;
3638                 }
3639         }
3640
3641         if (!match)
3642                 f = NULL;
3643         return (f);
3644 }
3645
3646 /*
3647 ** This routine takes additions to the vsi filter
3648 ** table and creates an Admin Queue call to create
3649 ** the filters in the hardware.
3650 */
3651 static void
3652 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
3653 {
3654         struct i40e_aqc_add_macvlan_element_data *a, *b;
3655         struct ixl_mac_filter   *f;
3656         struct ixl_pf           *pf;
3657         struct i40e_hw          *hw;
3658         device_t                dev;
3659         int                     err, j = 0;
3660
3661         pf = vsi->back;
3662         dev = pf->dev;
3663         hw = &pf->hw;
3664         IXL_PF_LOCK_ASSERT(pf);
3665
3666         a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
3667             M_DEVBUF, M_NOWAIT | M_ZERO);
3668         if (a == NULL) {
3669                 device_printf(dev, "add_hw_filters failed to get memory\n");
3670                 return;
3671         }
3672
3673         /*
3674         ** Scan the filter list, each time we find one
3675         ** we add it to the admin queue array and turn off
3676         ** the add bit.
3677         */
3678         SLIST_FOREACH(f, &vsi->ftl, next) {
3679                 if (f->flags == flags) {
3680                         b = &a[j]; // a pox on fvl long names :)
3681                         bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
3682                         if (f->vlan == IXL_VLAN_ANY) {
3683                                 b->vlan_tag = 0;
3684                                 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
3685                         } else {
3686                                 b->vlan_tag = f->vlan;
3687                                 b->flags = 0;
3688                         }
3689                         b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
3690                         f->flags &= ~IXL_FILTER_ADD;
3691                         j++;
3692                 }
3693                 if (j == cnt)
3694                         break;
3695         }
3696         if (j > 0) {
3697                 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
3698                 if (err)
3699                         device_printf(dev, "aq_add_macvlan err %d, "
3700                             "aq_error %d\n", err, hw->aq.asq_last_status);
3701                 else
3702                         vsi->hw_filters_add += j;
3703         }
3704         free(a, M_DEVBUF);
3705         return;
3706 }
3707
3708 /*
3709 ** This routine takes removals in the vsi filter
3710 ** table and creates an Admin Queue call to delete
3711 ** the filters in the hardware.
3712 */
3713 static void
3714 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
3715 {
3716         struct i40e_aqc_remove_macvlan_element_data *d, *e;
3717         struct ixl_pf           *pf;
3718         struct i40e_hw          *hw;
3719         device_t                dev;
3720         struct ixl_mac_filter   *f, *f_temp;
3721         int                     err, j = 0;
3722
3723         DEBUGOUT("ixl_del_hw_filters: begin\n");
3724
3725         pf = vsi->back;
3726         hw = &pf->hw;
3727         dev = pf->dev;
3728
3729         d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
3730             M_DEVBUF, M_NOWAIT | M_ZERO);
3731         if (d == NULL) {
3732                 printf("del hw filter failed to get memory\n");
3733                 return;
3734         }
3735
3736         SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
3737                 if (f->flags & IXL_FILTER_DEL) {
3738                         e = &d[j]; // a pox on fvl long names :)
3739                         bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
3740                         e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
3741                         e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3742                         /* delete entry from vsi list */
3743                         SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
3744                         free(f, M_DEVBUF);
3745                         j++;
3746                 }
3747                 if (j == cnt)
3748                         break;
3749         }
3750         if (j > 0) {
3751                 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
3752                 /* NOTE: returns ENOENT every time but seems to work fine,
3753                    so we'll ignore that specific error. */
3754                 // TODO: Does this still occur on current firmwares?
3755                 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
3756                         int sc = 0;
3757                         for (int i = 0; i < j; i++)
3758                                 sc += (!d[i].error_code);
3759                         vsi->hw_filters_del += sc;
3760                         device_printf(dev,
3761                             "Failed to remove %d/%d filters, aq error %d\n",
3762                             j - sc, j, hw->aq.asq_last_status);
3763                 } else
3764                         vsi->hw_filters_del += j;
3765         }
3766         free(d, M_DEVBUF);
3767
3768         DEBUGOUT("ixl_del_hw_filters: end\n");
3769         return;
3770 }
3771
3772 static int
3773 ixl_enable_rings(struct ixl_vsi *vsi)
3774 {
3775         struct ixl_pf   *pf = vsi->back;
3776         struct i40e_hw  *hw = &pf->hw;
3777         int             index, error;
3778         u32             reg;
3779
3780         error = 0;
3781         for (int i = 0; i < vsi->num_queues; i++) {
3782                 index = vsi->first_queue + i;
3783                 i40e_pre_tx_queue_cfg(hw, index, TRUE);
3784
3785                 reg = rd32(hw, I40E_QTX_ENA(index));
3786                 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3787                     I40E_QTX_ENA_QENA_STAT_MASK;
3788                 wr32(hw, I40E_QTX_ENA(index), reg);
3789                 /* Verify the enable took */
3790                 for (int j = 0; j < 10; j++) {
3791                         reg = rd32(hw, I40E_QTX_ENA(index));
3792                         if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
3793                                 break;
3794                         i40e_msec_delay(10);
3795                 }
3796                 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
3797                         device_printf(pf->dev, "TX queue %d disabled!\n",
3798                             index);
3799                         error = ETIMEDOUT;
3800                 }
3801
3802                 reg = rd32(hw, I40E_QRX_ENA(index));
3803                 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3804                     I40E_QRX_ENA_QENA_STAT_MASK;
3805                 wr32(hw, I40E_QRX_ENA(index), reg);
3806                 /* Verify the enable took */
3807                 for (int j = 0; j < 10; j++) {
3808                         reg = rd32(hw, I40E_QRX_ENA(index));
3809                         if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3810                                 break;
3811                         i40e_msec_delay(10);
3812                 }
3813                 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
3814                         device_printf(pf->dev, "RX queue %d disabled!\n",
3815                             index);
3816                         error = ETIMEDOUT;
3817                 }
3818         }
3819
3820         return (error);
3821 }
3822
3823 static int
3824 ixl_disable_rings(struct ixl_vsi *vsi)
3825 {
3826         struct ixl_pf   *pf = vsi->back;
3827         struct i40e_hw  *hw = &pf->hw;
3828         int             index, error;
3829         u32             reg;
3830
3831         error = 0;
3832         for (int i = 0; i < vsi->num_queues; i++) {
3833                 index = vsi->first_queue + i;
3834
3835                 i40e_pre_tx_queue_cfg(hw, index, FALSE);
3836                 i40e_usec_delay(500);
3837
3838                 reg = rd32(hw, I40E_QTX_ENA(index));
3839                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3840                 wr32(hw, I40E_QTX_ENA(index), reg);
3841                 /* Verify the disable took */
3842                 for (int j = 0; j < 10; j++) {
3843                         reg = rd32(hw, I40E_QTX_ENA(index));
3844                         if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3845                                 break;
3846                         i40e_msec_delay(10);
3847                 }
3848                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
3849                         device_printf(pf->dev, "TX queue %d still enabled!\n",
3850                             index);
3851                         error = ETIMEDOUT;
3852                 }
3853
3854                 reg = rd32(hw, I40E_QRX_ENA(index));
3855                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3856                 wr32(hw, I40E_QRX_ENA(index), reg);
3857                 /* Verify the disable took */
3858                 for (int j = 0; j < 10; j++) {
3859                         reg = rd32(hw, I40E_QRX_ENA(index));
3860                         if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3861                                 break;
3862                         i40e_msec_delay(10);
3863                 }
3864                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
3865                         device_printf(pf->dev, "RX queue %d still enabled!\n",
3866                             index);
3867                         error = ETIMEDOUT;
3868                 }
3869         }
3870
3871         return (error);
3872 }
3873
3874 /**
3875  * ixl_handle_mdd_event
3876  *
3877  * Called from interrupt handler to identify possibly malicious vfs
3878  * (But also detects events from the PF, as well)
3879  **/
3880 static void ixl_handle_mdd_event(struct ixl_pf *pf)
3881 {
3882         struct i40e_hw *hw = &pf->hw;
3883         device_t dev = pf->dev;
3884         bool mdd_detected = false;
3885         bool pf_mdd_detected = false;
3886         u32 reg;
3887
3888         /* find what triggered the MDD event */
3889         reg = rd32(hw, I40E_GL_MDET_TX);
3890         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
3891                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
3892                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
3893                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
3894                                 I40E_GL_MDET_TX_EVENT_SHIFT;
3895                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
3896                                 I40E_GL_MDET_TX_QUEUE_SHIFT;
3897                 device_printf(dev,
3898                          "Malicious Driver Detection event 0x%02x"
3899                          " on TX queue %d pf number 0x%02x\n",
3900                          event, queue, pf_num);
3901                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
3902                 mdd_detected = true;
3903         }
3904         reg = rd32(hw, I40E_GL_MDET_RX);
3905         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
3906                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
3907                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
3908                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
3909                                 I40E_GL_MDET_RX_EVENT_SHIFT;
3910                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
3911                                 I40E_GL_MDET_RX_QUEUE_SHIFT;
3912                 device_printf(dev,
3913                          "Malicious Driver Detection event 0x%02x"
3914                          " on RX queue %d of function 0x%02x\n",
3915                          event, queue, func);
3916                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
3917                 mdd_detected = true;
3918         }
3919
3920         if (mdd_detected) {
3921                 reg = rd32(hw, I40E_PF_MDET_TX);
3922                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
3923                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
3924                         device_printf(dev,
3925                                  "MDD TX event is for this function 0x%08x",
3926                                  reg);
3927                         pf_mdd_detected = true;
3928                 }
3929                 reg = rd32(hw, I40E_PF_MDET_RX);
3930                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
3931                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
3932                         device_printf(dev,
3933                                  "MDD RX event is for this function 0x%08x",
3934                                  reg);
3935                         pf_mdd_detected = true;
3936                 }
3937         }
3938
3939         /* re-enable mdd interrupt cause */
3940         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
3941         reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3942         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
3943         ixl_flush(hw);
3944 }
3945
3946 static void
3947 ixl_enable_intr(struct ixl_vsi *vsi)
3948 {
3949         struct i40e_hw          *hw = vsi->hw;
3950         struct ixl_queue        *que = vsi->queues;
3951
3952         if (ixl_enable_msix) {
3953                 ixl_enable_adminq(hw);
3954                 for (int i = 0; i < vsi->num_queues; i++, que++)
3955                         ixl_enable_queue(hw, que->me);
3956         } else
3957                 ixl_enable_legacy(hw);
3958 }
3959
3960 static void
3961 ixl_disable_rings_intr(struct ixl_vsi *vsi)
3962 {
3963         struct i40e_hw          *hw = vsi->hw;
3964         struct ixl_queue        *que = vsi->queues;
3965
3966         for (int i = 0; i < vsi->num_queues; i++, que++)
3967                 ixl_disable_queue(hw, que->me);
3968 }
3969
3970 static void
3971 ixl_disable_intr(struct ixl_vsi *vsi)
3972 {
3973         struct i40e_hw          *hw = vsi->hw;
3974
3975         if (ixl_enable_msix)
3976                 ixl_disable_adminq(hw);
3977         else
3978                 ixl_disable_legacy(hw);
3979 }
3980
3981 static void
3982 ixl_enable_adminq(struct i40e_hw *hw)
3983 {
3984         u32             reg;
3985
3986         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
3987             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3988             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3989         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
3990         ixl_flush(hw);
3991         return;
3992 }
3993
3994 static void
3995 ixl_disable_adminq(struct i40e_hw *hw)
3996 {
3997         u32             reg;
3998
3999         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4000         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4001
4002         return;
4003 }
4004
4005 static void
4006 ixl_enable_queue(struct i40e_hw *hw, int id)
4007 {
4008         u32             reg;
4009
4010         reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4011             I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4012             (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4013         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4014 }
4015
4016 static void
4017 ixl_disable_queue(struct i40e_hw *hw, int id)
4018 {
4019         u32             reg;
4020
4021         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4022         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4023
4024         return;
4025 }
4026
4027 static void
4028 ixl_enable_legacy(struct i40e_hw *hw)
4029 {
4030         u32             reg;
4031         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4032             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4033             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4034         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4035 }
4036
4037 static void
4038 ixl_disable_legacy(struct i40e_hw *hw)
4039 {
4040         u32             reg;
4041
4042         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4043         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4044
4045         return;
4046 }
4047
4048 static void
4049 ixl_update_stats_counters(struct ixl_pf *pf)
4050 {
4051         struct i40e_hw  *hw = &pf->hw;
4052         struct ixl_vsi  *vsi = &pf->vsi;
4053         struct ixl_vf   *vf;
4054
4055         struct i40e_hw_port_stats *nsd = &pf->stats;
4056         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4057
4058         /* Update hw stats */
4059         ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4060                            pf->stat_offsets_loaded,
4061                            &osd->crc_errors, &nsd->crc_errors);
4062         ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4063                            pf->stat_offsets_loaded,
4064                            &osd->illegal_bytes, &nsd->illegal_bytes);
4065         ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4066                            I40E_GLPRT_GORCL(hw->port),
4067                            pf->stat_offsets_loaded,
4068                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4069         ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4070                            I40E_GLPRT_GOTCL(hw->port),
4071                            pf->stat_offsets_loaded,
4072                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4073         ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4074                            pf->stat_offsets_loaded,
4075                            &osd->eth.rx_discards,
4076                            &nsd->eth.rx_discards);
4077         ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4078                            I40E_GLPRT_UPRCL(hw->port),
4079                            pf->stat_offsets_loaded,
4080                            &osd->eth.rx_unicast,
4081                            &nsd->eth.rx_unicast);
4082         ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4083                            I40E_GLPRT_UPTCL(hw->port),
4084                            pf->stat_offsets_loaded,
4085                            &osd->eth.tx_unicast,
4086                            &nsd->eth.tx_unicast);
4087         ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4088                            I40E_GLPRT_MPRCL(hw->port),
4089                            pf->stat_offsets_loaded,
4090                            &osd->eth.rx_multicast,
4091                            &nsd->eth.rx_multicast);
4092         ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4093                            I40E_GLPRT_MPTCL(hw->port),
4094                            pf->stat_offsets_loaded,
4095                            &osd->eth.tx_multicast,
4096                            &nsd->eth.tx_multicast);
4097         ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4098                            I40E_GLPRT_BPRCL(hw->port),
4099                            pf->stat_offsets_loaded,
4100                            &osd->eth.rx_broadcast,
4101                            &nsd->eth.rx_broadcast);
4102         ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4103                            I40E_GLPRT_BPTCL(hw->port),
4104                            pf->stat_offsets_loaded,
4105                            &osd->eth.tx_broadcast,
4106                            &nsd->eth.tx_broadcast);
4107
4108         ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4109                            pf->stat_offsets_loaded,
4110                            &osd->tx_dropped_link_down,
4111                            &nsd->tx_dropped_link_down);
4112         ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4113                            pf->stat_offsets_loaded,
4114                            &osd->mac_local_faults,
4115                            &nsd->mac_local_faults);
4116         ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4117                            pf->stat_offsets_loaded,
4118                            &osd->mac_remote_faults,
4119                            &nsd->mac_remote_faults);
4120         ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4121                            pf->stat_offsets_loaded,
4122                            &osd->rx_length_errors,
4123                            &nsd->rx_length_errors);
4124
4125         /* Flow control (LFC) stats */
4126         ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4127                            pf->stat_offsets_loaded,
4128                            &osd->link_xon_rx, &nsd->link_xon_rx);
4129         ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4130                            pf->stat_offsets_loaded,
4131                            &osd->link_xon_tx, &nsd->link_xon_tx);
4132         ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4133                            pf->stat_offsets_loaded,
4134                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
4135         ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4136                            pf->stat_offsets_loaded,
4137                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
4138
4139         /* Packet size stats rx */
4140         ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4141                            I40E_GLPRT_PRC64L(hw->port),
4142                            pf->stat_offsets_loaded,
4143                            &osd->rx_size_64, &nsd->rx_size_64);
4144         ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4145                            I40E_GLPRT_PRC127L(hw->port),
4146                            pf->stat_offsets_loaded,
4147                            &osd->rx_size_127, &nsd->rx_size_127);
4148         ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4149                            I40E_GLPRT_PRC255L(hw->port),
4150                            pf->stat_offsets_loaded,
4151                            &osd->rx_size_255, &nsd->rx_size_255);
4152         ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4153                            I40E_GLPRT_PRC511L(hw->port),
4154                            pf->stat_offsets_loaded,
4155                            &osd->rx_size_511, &nsd->rx_size_511);
4156         ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4157                            I40E_GLPRT_PRC1023L(hw->port),
4158                            pf->stat_offsets_loaded,
4159                            &osd->rx_size_1023, &nsd->rx_size_1023);
4160         ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4161                            I40E_GLPRT_PRC1522L(hw->port),
4162                            pf->stat_offsets_loaded,
4163                            &osd->rx_size_1522, &nsd->rx_size_1522);
4164         ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4165                            I40E_GLPRT_PRC9522L(hw->port),
4166                            pf->stat_offsets_loaded,
4167                            &osd->rx_size_big, &nsd->rx_size_big);
4168
4169         /* Packet size stats tx */
4170         ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4171                            I40E_GLPRT_PTC64L(hw->port),
4172                            pf->stat_offsets_loaded,
4173                            &osd->tx_size_64, &nsd->tx_size_64);
4174         ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4175                            I40E_GLPRT_PTC127L(hw->port),
4176                            pf->stat_offsets_loaded,
4177                            &osd->tx_size_127, &nsd->tx_size_127);
4178         ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4179                            I40E_GLPRT_PTC255L(hw->port),
4180                            pf->stat_offsets_loaded,
4181                            &osd->tx_size_255, &nsd->tx_size_255);
4182         ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4183                            I40E_GLPRT_PTC511L(hw->port),
4184                            pf->stat_offsets_loaded,
4185                            &osd->tx_size_511, &nsd->tx_size_511);
4186         ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4187                            I40E_GLPRT_PTC1023L(hw->port),
4188                            pf->stat_offsets_loaded,
4189                            &osd->tx_size_1023, &nsd->tx_size_1023);
4190         ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4191                            I40E_GLPRT_PTC1522L(hw->port),
4192                            pf->stat_offsets_loaded,
4193                            &osd->tx_size_1522, &nsd->tx_size_1522);
4194         ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4195                            I40E_GLPRT_PTC9522L(hw->port),
4196                            pf->stat_offsets_loaded,
4197                            &osd->tx_size_big, &nsd->tx_size_big);
4198
4199         ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4200                            pf->stat_offsets_loaded,
4201                            &osd->rx_undersize, &nsd->rx_undersize);
4202         ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4203                            pf->stat_offsets_loaded,
4204                            &osd->rx_fragments, &nsd->rx_fragments);
4205         ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4206                            pf->stat_offsets_loaded,
4207                            &osd->rx_oversize, &nsd->rx_oversize);
4208         ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4209                            pf->stat_offsets_loaded,
4210                            &osd->rx_jabber, &nsd->rx_jabber);
4211         pf->stat_offsets_loaded = true;
4212         /* End hw stats */
4213
4214         /* Update vsi stats */
4215         ixl_update_vsi_stats(vsi);
4216
4217         for (int i = 0; i < pf->num_vfs; i++) {
4218                 vf = &pf->vfs[i];
4219                 if (vf->vf_flags & VF_FLAG_ENABLED)
4220                         ixl_update_eth_stats(&pf->vfs[i].vsi);
4221         }
4222 }
4223
4224 /*
4225 ** Tasklet handler for MSIX Adminq interrupts
4226 **  - do outside interrupt since it might sleep
4227 */
4228 static void
4229 ixl_do_adminq(void *context, int pending)
4230 {
4231         struct ixl_pf                   *pf = context;
4232         struct i40e_hw                  *hw = &pf->hw;
4233         struct ixl_vsi                  *vsi = &pf->vsi;
4234         struct i40e_arq_event_info      event;
4235         i40e_status                     ret;
4236         u32                             reg, loop = 0;
4237         u16                             opcode, result;
4238
4239         event.buf_len = IXL_AQ_BUF_SZ;
4240         event.msg_buf = malloc(event.buf_len,
4241             M_DEVBUF, M_NOWAIT | M_ZERO);
4242         if (!event.msg_buf) {
4243                 printf("Unable to allocate adminq memory\n");
4244                 return;
4245         }
4246
4247         IXL_PF_LOCK(pf);
4248         /* clean and process any events */
4249         do {
4250                 ret = i40e_clean_arq_element(hw, &event, &result);
4251                 if (ret)
4252                         break;
4253                 opcode = LE16_TO_CPU(event.desc.opcode);
4254                 switch (opcode) {
4255                 case i40e_aqc_opc_get_link_status:
4256                         ixl_link_event(pf, &event);
4257                         ixl_update_link_status(pf);
4258                         break;
4259                 case i40e_aqc_opc_send_msg_to_pf:
4260 #ifdef PCI_IOV
4261                         ixl_handle_vf_msg(pf, &event);
4262 #endif
4263                         break;
4264                 case i40e_aqc_opc_event_lan_overflow:
4265                         break;
4266                 default:
4267 #ifdef IXL_DEBUG
4268                         printf("AdminQ unknown event %x\n", opcode);
4269 #endif
4270                         break;
4271                 }
4272
4273         } while (result && (loop++ < IXL_ADM_LIMIT));
4274
4275         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4276         reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4277         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4278         free(event.msg_buf, M_DEVBUF);
4279
4280         /*
4281          * If there are still messages to process, reschedule ourselves.
4282          * Otherwise, re-enable our interrupt and go to sleep.
4283          */
4284         if (result > 0)
4285                 taskqueue_enqueue(pf->tq, &pf->adminq);
4286         else
4287                 ixl_enable_intr(vsi);
4288
4289         IXL_PF_UNLOCK(pf);
4290 }
4291
4292 static int
4293 ixl_debug_info(SYSCTL_HANDLER_ARGS)
4294 {
4295         struct ixl_pf   *pf;
4296         int             error, input = 0;
4297
4298         error = sysctl_handle_int(oidp, &input, 0, req);
4299
4300         if (error || !req->newptr)
4301                 return (error);
4302
4303         if (input == 1) {
4304                 pf = (struct ixl_pf *)arg1;
4305                 ixl_print_debug_info(pf);
4306         }
4307
4308         return (error);
4309 }
4310
4311 static void
4312 ixl_print_debug_info(struct ixl_pf *pf)
4313 {
4314         struct i40e_hw          *hw = &pf->hw;
4315         struct ixl_vsi          *vsi = &pf->vsi;
4316         struct ixl_queue        *que = vsi->queues;
4317         struct rx_ring          *rxr = &que->rxr;
4318         struct tx_ring          *txr = &que->txr;
4319         u32                     reg;
4320
4321
4322         printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
4323         printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
4324         printf("RX next check = %x\n", rxr->next_check);
4325         printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
4326         printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
4327         printf("TX desc avail = %x\n", txr->avail);
4328
4329         reg = rd32(hw, I40E_GLV_GORCL(0xc));
4330          printf("RX Bytes = %x\n", reg);
4331         reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
4332          printf("Port RX Bytes = %x\n", reg);
4333         reg = rd32(hw, I40E_GLV_RDPC(0xc));
4334          printf("RX discard = %x\n", reg);
4335         reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
4336          printf("Port RX discard = %x\n", reg);
4337
4338         reg = rd32(hw, I40E_GLV_TEPC(0xc));
4339          printf("TX errors = %x\n", reg);
4340         reg = rd32(hw, I40E_GLV_GOTCL(0xc));
4341          printf("TX Bytes = %x\n", reg);
4342
4343         reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
4344          printf("RX undersize = %x\n", reg);
4345         reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
4346          printf("RX fragments = %x\n", reg);
4347         reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
4348          printf("RX oversize = %x\n", reg);
4349         reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
4350          printf("RX length error = %x\n", reg);
4351         reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
4352          printf("mac remote fault = %x\n", reg);
4353         reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
4354          printf("mac local fault = %x\n", reg);
4355 }
4356
4357 /**
4358  * Update VSI-specific ethernet statistics counters.
4359  **/
4360 void ixl_update_eth_stats(struct ixl_vsi *vsi)
4361 {
4362         struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4363         struct i40e_hw *hw = &pf->hw;
4364         struct i40e_eth_stats *es;
4365         struct i40e_eth_stats *oes;
4366         struct i40e_hw_port_stats *nsd;
4367         u16 stat_idx = vsi->info.stat_counter_idx;
4368
4369         es = &vsi->eth_stats;
4370         oes = &vsi->eth_stats_offsets;
4371         nsd = &pf->stats;
4372
4373         /* Gather up the stats that the hw collects */
4374         ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4375                            vsi->stat_offsets_loaded,
4376                            &oes->tx_errors, &es->tx_errors);
4377         ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4378                            vsi->stat_offsets_loaded,
4379                            &oes->rx_discards, &es->rx_discards);
4380
4381         ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4382                            I40E_GLV_GORCL(stat_idx),
4383                            vsi->stat_offsets_loaded,
4384                            &oes->rx_bytes, &es->rx_bytes);
4385         ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4386                            I40E_GLV_UPRCL(stat_idx),
4387                            vsi->stat_offsets_loaded,
4388                            &oes->rx_unicast, &es->rx_unicast);
4389         ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4390                            I40E_GLV_MPRCL(stat_idx),
4391                            vsi->stat_offsets_loaded,
4392                            &oes->rx_multicast, &es->rx_multicast);
4393         ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4394                            I40E_GLV_BPRCL(stat_idx),
4395                            vsi->stat_offsets_loaded,
4396                            &oes->rx_broadcast, &es->rx_broadcast);
4397
4398         ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4399                            I40E_GLV_GOTCL(stat_idx),
4400                            vsi->stat_offsets_loaded,
4401                            &oes->tx_bytes, &es->tx_bytes);
4402         ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4403                            I40E_GLV_UPTCL(stat_idx),
4404                            vsi->stat_offsets_loaded,
4405                            &oes->tx_unicast, &es->tx_unicast);
4406         ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4407                            I40E_GLV_MPTCL(stat_idx),
4408                            vsi->stat_offsets_loaded,
4409                            &oes->tx_multicast, &es->tx_multicast);
4410         ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4411                            I40E_GLV_BPTCL(stat_idx),
4412                            vsi->stat_offsets_loaded,
4413                            &oes->tx_broadcast, &es->tx_broadcast);
4414         vsi->stat_offsets_loaded = true;
4415 }
4416
4417 static void
4418 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4419 {
4420         struct ixl_pf           *pf;
4421         struct ifnet            *ifp;
4422         struct i40e_eth_stats   *es;
4423         u64                     tx_discards;
4424
4425         struct i40e_hw_port_stats *nsd;
4426
4427         pf = vsi->back;
4428         ifp = vsi->ifp;
4429         es = &vsi->eth_stats;
4430         nsd = &pf->stats;
4431
4432         ixl_update_eth_stats(vsi);
4433
4434         tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4435         for (int i = 0; i < vsi->num_queues; i++)
4436                 tx_discards += vsi->queues[i].txr.br->br_drops;
4437
4438         /* Update ifnet stats */
4439         IXL_SET_IPACKETS(vsi, es->rx_unicast +
4440                            es->rx_multicast +
4441                            es->rx_broadcast);
4442         IXL_SET_OPACKETS(vsi, es->tx_unicast +
4443                            es->tx_multicast +
4444                            es->tx_broadcast);
4445         IXL_SET_IBYTES(vsi, es->rx_bytes);
4446         IXL_SET_OBYTES(vsi, es->tx_bytes);
4447         IXL_SET_IMCASTS(vsi, es->rx_multicast);
4448         IXL_SET_OMCASTS(vsi, es->tx_multicast);
4449
4450         IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
4451             nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
4452             nsd->rx_jabber);
4453         IXL_SET_OERRORS(vsi, es->tx_errors);
4454         IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
4455         IXL_SET_OQDROPS(vsi, tx_discards);
4456         IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
4457         IXL_SET_COLLISIONS(vsi, 0);
4458 }
4459
4460 /**
4461  * Reset all of the stats for the given pf
4462  **/
4463 void ixl_pf_reset_stats(struct ixl_pf *pf)
4464 {
4465         bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
4466         bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
4467         pf->stat_offsets_loaded = false;
4468 }
4469
4470 /**
4471  * Resets all stats of the given vsi
4472  **/
4473 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
4474 {
4475         bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
4476         bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
4477         vsi->stat_offsets_loaded = false;
4478 }
4479
4480 /**
4481  * Read and update a 48 bit stat from the hw
4482  *
4483  * Since the device stats are not reset at PFReset, they likely will not
4484  * be zeroed when the driver starts.  We'll save the first values read
4485  * and use them as offsets to be subtracted from the raw values in order
4486  * to report stats that count from zero.
4487  **/
4488 static void
4489 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
4490         bool offset_loaded, u64 *offset, u64 *stat)
4491 {
4492         u64 new_data;
4493
4494 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
4495         new_data = rd64(hw, loreg);
4496 #else
4497         /*
4498          * Use two rd32's instead of one rd64; FreeBSD versions before
4499          * 10 don't support 8 byte bus reads/writes.
4500          */
4501         new_data = rd32(hw, loreg);
4502         new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4503 #endif
4504
4505         if (!offset_loaded)
4506                 *offset = new_data;
4507         if (new_data >= *offset)
4508                 *stat = new_data - *offset;
4509         else
4510                 *stat = (new_data + ((u64)1 << 48)) - *offset;
4511         *stat &= 0xFFFFFFFFFFFFULL;
4512 }
4513
4514 /**
4515  * Read and update a 32 bit stat from the hw
4516  **/
4517 static void
4518 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
4519         bool offset_loaded, u64 *offset, u64 *stat)
4520 {
4521         u32 new_data;
4522
4523         new_data = rd32(hw, reg);
4524         if (!offset_loaded)
4525                 *offset = new_data;
4526         if (new_data >= *offset)
4527                 *stat = (u32)(new_data - *offset);
4528         else
4529                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
4530 }
4531
4532 /*
4533 ** Set flow control using sysctl:
4534 **      0 - off
4535 **      1 - rx pause
4536 **      2 - tx pause
4537 **      3 - full
4538 */
4539 static int
4540 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
4541 {
4542         /*
4543          * TODO: ensure flow control is disabled if
4544          * priority flow control is enabled
4545          *
4546          * TODO: ensure tx CRC by hardware should be enabled
4547          * if tx flow control is enabled.
4548          */
4549         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4550         struct i40e_hw *hw = &pf->hw;
4551         device_t dev = pf->dev;
4552         int error = 0;
4553         enum i40e_status_code aq_error = 0;
4554         u8 fc_aq_err = 0;
4555
4556         /* Get request */
4557         error = sysctl_handle_int(oidp, &pf->fc, 0, req);
4558         if ((error) || (req->newptr == NULL))
4559                 return (error);
4560         if (pf->fc < 0 || pf->fc > 3) {
4561                 device_printf(dev,
4562                     "Invalid fc mode; valid modes are 0 through 3\n");
4563                 return (EINVAL);
4564         }
4565
4566         /*
4567         ** Changing flow control mode currently does not work on
4568         ** 40GBASE-CR4 PHYs
4569         */
4570         if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4
4571             || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) {
4572                 device_printf(dev, "Changing flow control mode unsupported"
4573                     " on 40GBase-CR4 media.\n");
4574                 return (ENODEV);
4575         }
4576
4577         /* Set fc ability for port */
4578         hw->fc.requested_mode = pf->fc;
4579         aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
4580         if (aq_error) {
4581                 device_printf(dev,
4582                     "%s: Error setting new fc mode %d; fc_err %#x\n",
4583                     __func__, aq_error, fc_aq_err);
4584                 return (EAGAIN);
4585         }
4586
4587         return (0);
4588 }
4589
4590 static int
4591 ixl_current_speed(SYSCTL_HANDLER_ARGS)
4592 {
4593         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4594         struct i40e_hw *hw = &pf->hw;
4595         int error = 0, index = 0;
4596
4597         char *speeds[] = {
4598                 "Unknown",
4599                 "100M",
4600                 "1G",
4601                 "10G",
4602                 "40G",
4603                 "20G"
4604         };
4605
4606         ixl_update_link_status(pf);
4607
4608         switch (hw->phy.link_info.link_speed) {
4609         case I40E_LINK_SPEED_100MB:
4610                 index = 1;
4611                 break;
4612         case I40E_LINK_SPEED_1GB:
4613                 index = 2;
4614                 break;
4615         case I40E_LINK_SPEED_10GB:
4616                 index = 3;
4617                 break;
4618         case I40E_LINK_SPEED_40GB:
4619                 index = 4;
4620                 break;
4621         case I40E_LINK_SPEED_20GB:
4622                 index = 5;
4623                 break;
4624         case I40E_LINK_SPEED_UNKNOWN:
4625         default:
4626                 index = 0;
4627                 break;
4628         }
4629
4630         error = sysctl_handle_string(oidp, speeds[index],
4631             strlen(speeds[index]), req);
4632         return (error);
4633 }
4634
4635 static int
4636 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
4637 {
4638         struct i40e_hw *hw = &pf->hw;
4639         device_t dev = pf->dev;
4640         struct i40e_aq_get_phy_abilities_resp abilities;
4641         struct i40e_aq_set_phy_config config;
4642         enum i40e_status_code aq_error = 0;
4643
4644         /* Get current capability information */
4645         aq_error = i40e_aq_get_phy_capabilities(hw,
4646             FALSE, FALSE, &abilities, NULL);
4647         if (aq_error) {
4648                 device_printf(dev,
4649                     "%s: Error getting phy capabilities %d,"
4650                     " aq error: %d\n", __func__, aq_error,
4651                     hw->aq.asq_last_status);
4652                 return (EAGAIN);
4653         }
4654
4655         /* Prepare new config */
4656         bzero(&config, sizeof(config));
4657         config.phy_type = abilities.phy_type;
4658         config.abilities = abilities.abilities
4659             | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
4660         config.eee_capability = abilities.eee_capability;
4661         config.eeer = abilities.eeer_val;
4662         config.low_power_ctrl = abilities.d3_lpan;
4663         /* Translate into aq cmd link_speed */
4664         if (speeds & 0x8)
4665                 config.link_speed |= I40E_LINK_SPEED_20GB;
4666         if (speeds & 0x4)
4667                 config.link_speed |= I40E_LINK_SPEED_10GB;
4668         if (speeds & 0x2)
4669                 config.link_speed |= I40E_LINK_SPEED_1GB;
4670         if (speeds & 0x1)
4671                 config.link_speed |= I40E_LINK_SPEED_100MB;
4672
4673         /* Do aq command & restart link */
4674         aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
4675         if (aq_error) {
4676                 device_printf(dev,
4677                     "%s: Error setting new phy config %d,"
4678                     " aq error: %d\n", __func__, aq_error,
4679                     hw->aq.asq_last_status);
4680                 return (EAGAIN);
4681         }
4682
4683         /*
4684         ** This seems a bit heavy handed, but we
4685         ** need to get a reinit on some devices
4686         */
4687         IXL_PF_LOCK(pf);
4688         ixl_stop(pf);
4689         ixl_init_locked(pf);
4690         IXL_PF_UNLOCK(pf);
4691
4692         return (0);
4693 }
4694
4695 /*
4696 ** Control link advertise speed:
4697 **      Flags:
4698 **      0x1 - advertise 100 Mb
4699 **      0x2 - advertise 1G
4700 **      0x4 - advertise 10G
4701 **      0x8 - advertise 20G
4702 **
4703 ** Does not work on 40G devices.
4704 */
4705 static int
4706 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
4707 {
4708         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4709         struct i40e_hw *hw = &pf->hw;
4710         device_t dev = pf->dev;
4711         int requested_ls = 0;
4712         int error = 0;
4713
4714         /*
4715         ** FW doesn't support changing advertised speed
4716         ** for 40G devices; speed is always 40G.
4717         */
4718         if (i40e_is_40G_device(hw->device_id))
4719                 return (ENODEV);
4720
4721         /* Read in new mode */
4722         requested_ls = pf->advertised_speed;
4723         error = sysctl_handle_int(oidp, &requested_ls, 0, req);
4724         if ((error) || (req->newptr == NULL))
4725                 return (error);
4726         /* Check for sane value */
4727         if (requested_ls < 0x1 || requested_ls > 0xE) {
4728                 device_printf(dev, "Invalid advertised speed; "
4729                     "valid modes are 0x1 through 0xE\n");
4730                 return (EINVAL);
4731         }
4732         /* Then check for validity based on adapter type */
4733         switch (hw->device_id) {
4734         case I40E_DEV_ID_10G_BASE_T:
4735                 if (requested_ls & 0x8) {
4736                         device_printf(dev,
4737                             "20Gbs speed not supported on this device.\n");
4738                         return (EINVAL);
4739                 }
4740                 break;
4741         case I40E_DEV_ID_20G_KR2:
4742                 if (requested_ls & 0x1) {
4743                         device_printf(dev,
4744                             "100Mbs speed not supported on this device.\n");
4745                         return (EINVAL);
4746                 }
4747                 break;
4748         default:
4749                 if (requested_ls & ~0x6) {
4750                         device_printf(dev,
4751                             "Only 1/10Gbs speeds are supported on this device.\n");
4752                         return (EINVAL);
4753                 }
4754                 break;
4755         }
4756
4757         /* Exit if no change */
4758         if (pf->advertised_speed == requested_ls)
4759                 return (0);
4760
4761         error = ixl_set_advertised_speeds(pf, requested_ls);
4762         if (error)
4763                 return (error);
4764
4765         pf->advertised_speed = requested_ls;
4766         ixl_update_link_status(pf);
4767         return (0);
4768 }
4769
4770 /*
4771 ** Get the width and transaction speed of
4772 ** the bus this adapter is plugged into.
4773 */
4774 static u16
4775 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
4776 {
4777         u16                     link;
4778         u32                     offset;
4779
4780
4781         /* Get the PCI Express Capabilities offset */
4782         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4783
4784         /* ...and read the Link Status Register */
4785         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4786
4787         switch (link & I40E_PCI_LINK_WIDTH) {
4788         case I40E_PCI_LINK_WIDTH_1:
4789                 hw->bus.width = i40e_bus_width_pcie_x1;
4790                 break;
4791         case I40E_PCI_LINK_WIDTH_2:
4792                 hw->bus.width = i40e_bus_width_pcie_x2;
4793                 break;
4794         case I40E_PCI_LINK_WIDTH_4:
4795                 hw->bus.width = i40e_bus_width_pcie_x4;
4796                 break;
4797         case I40E_PCI_LINK_WIDTH_8:
4798                 hw->bus.width = i40e_bus_width_pcie_x8;
4799                 break;
4800         default:
4801                 hw->bus.width = i40e_bus_width_unknown;
4802                 break;
4803         }
4804
4805         switch (link & I40E_PCI_LINK_SPEED) {
4806         case I40E_PCI_LINK_SPEED_2500:
4807                 hw->bus.speed = i40e_bus_speed_2500;
4808                 break;
4809         case I40E_PCI_LINK_SPEED_5000:
4810                 hw->bus.speed = i40e_bus_speed_5000;
4811                 break;
4812         case I40E_PCI_LINK_SPEED_8000:
4813                 hw->bus.speed = i40e_bus_speed_8000;
4814                 break;
4815         default:
4816                 hw->bus.speed = i40e_bus_speed_unknown;
4817                 break;
4818         }
4819
4820
4821         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4822             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
4823             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
4824             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4825             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
4826             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
4827             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
4828             ("Unknown"));
4829
4830         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
4831             (hw->bus.speed < i40e_bus_speed_8000)) {
4832                 device_printf(dev, "PCI-Express bandwidth available"
4833                     " for this device\n     may be insufficient for"
4834                     " optimal performance.\n");
4835                 device_printf(dev, "For expected performance a x8 "
4836                     "PCIE Gen3 slot is required.\n");
4837         }
4838
4839         return (link);
4840 }
4841
4842 static int
4843 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
4844 {
4845         struct ixl_pf   *pf = (struct ixl_pf *)arg1;
4846         struct i40e_hw  *hw = &pf->hw;
4847         char            buf[32];
4848
4849         snprintf(buf, sizeof(buf),
4850             "f%d.%d a%d.%d n%02x.%02x e%08x",
4851             hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
4852             hw->aq.api_maj_ver, hw->aq.api_min_ver,
4853             (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >>
4854             IXL_NVM_VERSION_HI_SHIFT,
4855             (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >>
4856             IXL_NVM_VERSION_LO_SHIFT,
4857             hw->nvm.eetrack);
4858         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4859 }
4860
4861
4862 #ifdef IXL_DEBUG_SYSCTL
4863 static int
4864 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
4865 {
4866         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4867         struct i40e_hw *hw = &pf->hw;
4868         struct i40e_link_status link_status;
4869         char buf[512];
4870
4871         enum i40e_status_code aq_error = 0;
4872
4873         aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
4874         if (aq_error) {
4875                 printf("i40e_aq_get_link_info() error %d\n", aq_error);
4876                 return (EPERM);
4877         }
4878
4879         sprintf(buf, "\n"
4880             "PHY Type : %#04x\n"
4881             "Speed    : %#04x\n"
4882             "Link info: %#04x\n"
4883             "AN info  : %#04x\n"
4884             "Ext info : %#04x",
4885             link_status.phy_type, link_status.link_speed,
4886             link_status.link_info, link_status.an_info,
4887             link_status.ext_info);
4888
4889         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4890 }
4891
4892 static int
4893 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
4894 {
4895         struct ixl_pf           *pf = (struct ixl_pf *)arg1;
4896         struct i40e_hw          *hw = &pf->hw;
4897         char                    buf[512];
4898         enum i40e_status_code   aq_error = 0;
4899
4900         struct i40e_aq_get_phy_abilities_resp abilities;
4901
4902         aq_error = i40e_aq_get_phy_capabilities(hw,
4903             TRUE, FALSE, &abilities, NULL);
4904         if (aq_error) {
4905                 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
4906                 return (EPERM);
4907         }
4908
4909         sprintf(buf, "\n"
4910             "PHY Type : %#010x\n"
4911             "Speed    : %#04x\n"
4912             "Abilities: %#04x\n"
4913             "EEE cap  : %#06x\n"
4914             "EEER reg : %#010x\n"
4915             "D3 Lpan  : %#04x",
4916             abilities.phy_type, abilities.link_speed,
4917             abilities.abilities, abilities.eee_capability,
4918             abilities.eeer_val, abilities.d3_lpan);
4919
4920         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4921 }
4922
4923 static int
4924 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
4925 {
4926         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4927         struct ixl_vsi *vsi = &pf->vsi;
4928         struct ixl_mac_filter *f;
4929         char *buf, *buf_i;
4930
4931         int error = 0;
4932         int ftl_len = 0;
4933         int ftl_counter = 0;
4934         int buf_len = 0;
4935         int entry_len = 42;
4936
4937         SLIST_FOREACH(f, &vsi->ftl, next) {
4938                 ftl_len++;
4939         }
4940
4941         if (ftl_len < 1) {
4942                 sysctl_handle_string(oidp, "(none)", 6, req);
4943                 return (0);
4944         }
4945
4946         buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
4947         buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
4948
4949         sprintf(buf_i++, "\n");
4950         SLIST_FOREACH(f, &vsi->ftl, next) {
4951                 sprintf(buf_i,
4952                     MAC_FORMAT ", vlan %4d, flags %#06x",
4953                     MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
4954                 buf_i += entry_len;
4955                 /* don't print '\n' for last entry */
4956                 if (++ftl_counter != ftl_len) {
4957                         sprintf(buf_i, "\n");
4958                         buf_i++;
4959                 }
4960         }
4961
4962         error = sysctl_handle_string(oidp, buf, strlen(buf), req);
4963         if (error)
4964                 printf("sysctl error: %d\n", error);
4965         free(buf, M_DEVBUF);
4966         return error;
4967 }
4968
4969 #define IXL_SW_RES_SIZE 0x14
4970 static int
4971 ixl_res_alloc_cmp(const void *a, const void *b)
4972 {
4973         const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
4974         one = (struct i40e_aqc_switch_resource_alloc_element_resp *)a;
4975         two = (struct i40e_aqc_switch_resource_alloc_element_resp *)b;
4976
4977         return ((int)one->resource_type - (int)two->resource_type);
4978 }
4979
4980 static int
4981 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
4982 {
4983         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4984         struct i40e_hw *hw = &pf->hw;
4985         device_t dev = pf->dev;
4986         struct sbuf *buf;
4987         int error = 0;
4988
4989         u8 num_entries;
4990         struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
4991
4992         buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
4993         if (!buf) {
4994                 device_printf(dev, "Could not allocate sbuf for output.\n");
4995                 return (ENOMEM);
4996         }
4997
4998         bzero(resp, sizeof(resp));
4999         error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5000                                 resp,
5001                                 IXL_SW_RES_SIZE,
5002                                 NULL);
5003         if (error) {
5004                 device_printf(dev,
5005                     "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5006                     __func__, error, hw->aq.asq_last_status);
5007                 sbuf_delete(buf);
5008                 return error;
5009         }
5010
5011         /* Sort entries by type for display */
5012         qsort(resp, num_entries,
5013             sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5014             &ixl_res_alloc_cmp);
5015
5016         sbuf_cat(buf, "\n");
5017         sbuf_printf(buf, "# of entries: %d\n", num_entries);
5018         sbuf_printf(buf,
5019             "Type | Guaranteed | Total | Used   | Un-allocated\n"
5020             "     | (this)     | (all) | (this) | (all)       \n");
5021         for (int i = 0; i < num_entries; i++) {
5022                 sbuf_printf(buf,
5023                     "%#4x | %10d   %5d   %6d   %12d",
5024                     resp[i].resource_type,
5025                     resp[i].guaranteed,
5026                     resp[i].total,
5027                     resp[i].used,
5028                     resp[i].total_unalloced);
5029                 if (i < num_entries - 1)
5030                         sbuf_cat(buf, "\n");
5031         }
5032
5033         error = sbuf_finish(buf);
5034         sbuf_delete(buf);
5035
5036         return (error);
5037 }
5038
5039 /*
5040 ** Caller must init and delete sbuf; this function will clear and
5041 ** finish it for caller.
5042 */
5043 static char *
5044 ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink)
5045 {
5046         sbuf_clear(s);
5047
5048         if (seid == 0 && uplink)
5049                 sbuf_cat(s, "Network");
5050         else if (seid == 0)
5051                 sbuf_cat(s, "Host");
5052         else if (seid == 1)
5053                 sbuf_cat(s, "EMP");
5054         else if (seid <= 5)
5055                 sbuf_printf(s, "MAC %d", seid - 2);
5056         else if (seid <= 15)
5057                 sbuf_cat(s, "Reserved");
5058         else if (seid <= 31)
5059                 sbuf_printf(s, "PF %d", seid - 16);
5060         else if (seid <= 159)
5061                 sbuf_printf(s, "VF %d", seid - 32);
5062         else if (seid <= 287)
5063                 sbuf_cat(s, "Reserved");
5064         else if (seid <= 511)
5065                 sbuf_cat(s, "Other"); // for other structures
5066         else if (seid <= 895)
5067                 sbuf_printf(s, "VSI %d", seid - 512);
5068         else if (seid <= 1023)
5069                 sbuf_printf(s, "Reserved");
5070         else
5071                 sbuf_cat(s, "Invalid");
5072
5073         sbuf_finish(s);
5074         return sbuf_data(s);
5075 }
5076
5077 static int
5078 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5079 {
5080         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5081         struct i40e_hw *hw = &pf->hw;
5082         device_t dev = pf->dev;
5083         struct sbuf *buf;
5084         struct sbuf *nmbuf;
5085         int error = 0;
5086         u8 aq_buf[I40E_AQ_LARGE_BUF];
5087
5088         u16 next = 0;
5089         struct i40e_aqc_get_switch_config_resp *sw_config;
5090         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5091
5092         buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
5093         if (!buf) {
5094                 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5095                 return (ENOMEM);
5096         }
5097
5098         error = i40e_aq_get_switch_config(hw, sw_config,
5099             sizeof(aq_buf), &next, NULL);
5100         if (error) {
5101                 device_printf(dev,
5102                     "%s: aq_get_switch_config() error %d, aq error %d\n",
5103                     __func__, error, hw->aq.asq_last_status);
5104                 sbuf_delete(buf);
5105                 return error;
5106         }
5107
5108         nmbuf = sbuf_new_auto();
5109         if (!nmbuf) {
5110                 device_printf(dev, "Could not allocate sbuf for name output.\n");
5111                 return (ENOMEM);
5112         }
5113
5114         sbuf_cat(buf, "\n");
5115         // Assuming <= 255 elements in switch
5116         sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported);
5117         /* Exclude:
5118         ** Revision -- all elements are revision 1 for now
5119         */
5120         sbuf_printf(buf,
5121             "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5122             "                |          |          | (uplink)\n");
5123         for (int i = 0; i < sw_config->header.num_reported; i++) {
5124                 // "%4d (%8s) | %8s   %8s   %#8x",
5125                 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5126                 sbuf_cat(buf, " ");
5127                 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5128                     sw_config->element[i].seid, false));
5129                 sbuf_cat(buf, " | ");
5130                 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5131                     sw_config->element[i].uplink_seid, true));
5132                 sbuf_cat(buf, "   ");
5133                 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5134                     sw_config->element[i].downlink_seid, false));
5135                 sbuf_cat(buf, "   ");
5136                 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5137                 if (i < sw_config->header.num_reported - 1)
5138                         sbuf_cat(buf, "\n");
5139         }
5140         sbuf_delete(nmbuf);
5141
5142         error = sbuf_finish(buf);
5143         sbuf_delete(buf);
5144
5145         return (error);
5146 }
5147 #endif /* IXL_DEBUG_SYSCTL */
5148
5149
5150 #ifdef PCI_IOV
5151 static int
5152 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5153 {
5154         struct i40e_hw *hw;
5155         struct ixl_vsi *vsi;
5156         struct i40e_vsi_context vsi_ctx;
5157         int i;
5158         uint16_t first_queue;
5159         enum i40e_status_code code;
5160
5161         hw = &pf->hw;
5162         vsi = &pf->vsi;
5163
5164         vsi_ctx.pf_num = hw->pf_id;
5165         vsi_ctx.uplink_seid = pf->veb_seid;
5166         vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
5167         vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5168         vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
5169
5170         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
5171
5172         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5173         vsi_ctx.info.switch_id = htole16(0);
5174
5175         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
5176         vsi_ctx.info.sec_flags = 0;
5177         if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
5178                 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
5179
5180         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
5181         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
5182             I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5183
5184         vsi_ctx.info.valid_sections |=
5185             htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
5186         vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
5187         first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
5188         for (i = 0; i < IXLV_MAX_QUEUES; i++)
5189                 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
5190         for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
5191                 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
5192
5193         vsi_ctx.info.tc_mapping[0] = htole16(
5194             (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5195             (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
5196
5197         code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
5198         if (code != I40E_SUCCESS)
5199                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5200         vf->vsi.seid = vsi_ctx.seid;
5201         vf->vsi.vsi_num = vsi_ctx.vsi_number;
5202         vf->vsi.first_queue = first_queue;
5203         vf->vsi.num_queues = IXLV_MAX_QUEUES;
5204
5205         code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
5206         if (code != I40E_SUCCESS)
5207                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5208
5209         code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
5210         if (code != I40E_SUCCESS) {
5211                 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
5212                     ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5213                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5214         }
5215
5216         memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
5217         return (0);
5218 }
5219
5220 static int
5221 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5222 {
5223         struct i40e_hw *hw;
5224         int error;
5225
5226         hw = &pf->hw;
5227
5228         error = ixl_vf_alloc_vsi(pf, vf);
5229         if (error != 0)
5230                 return (error);
5231
5232         vf->vsi.hw_filters_add = 0;
5233         vf->vsi.hw_filters_del = 0;
5234         ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
5235         ixl_reconfigure_filters(&vf->vsi);
5236
5237         return (0);
5238 }
5239
5240 static void
5241 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
5242     uint32_t val)
5243 {
5244         uint32_t qtable;
5245         int index, shift;
5246
5247         /*
5248          * Two queues are mapped in a single register, so we have to do some
5249          * gymnastics to convert the queue number into a register index and
5250          * shift.
5251          */
5252         index = qnum / 2;
5253         shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
5254
5255         qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
5256         qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
5257         qtable |= val << shift;
5258         wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
5259 }
5260
5261 static void
5262 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
5263 {
5264         struct i40e_hw *hw;
5265         uint32_t qtable;
5266         int i;
5267
5268         hw = &pf->hw;
5269
5270         /*
5271          * Contiguous mappings aren't actually supported by the hardware,
5272          * so we have to use non-contiguous mappings.
5273          */
5274         wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
5275              I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
5276
5277         wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
5278             I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
5279
5280         for (i = 0; i < vf->vsi.num_queues; i++) {
5281                 qtable = (vf->vsi.first_queue + i) <<
5282                     I40E_VPLAN_QTABLE_QINDEX_SHIFT;
5283
5284                 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
5285         }
5286
5287         /* Map queues allocated to VF to its VSI. */
5288         for (i = 0; i < vf->vsi.num_queues; i++)
5289                 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
5290
5291         /* Set rest of VSI queues as unused. */
5292         for (; i < IXL_MAX_VSI_QUEUES; i++)
5293                 ixl_vf_map_vsi_queue(hw, vf, i,
5294                     I40E_VSILAN_QTABLE_QINDEX_0_MASK);
5295
5296         ixl_flush(hw);
5297 }
5298
5299 static void
5300 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
5301 {
5302         struct i40e_hw *hw;
5303
5304         hw = &pf->hw;
5305
5306         if (vsi->seid == 0)
5307                 return;
5308
5309         i40e_aq_delete_element(hw, vsi->seid, NULL);
5310 }
5311
5312 static void
5313 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
5314 {
5315
5316         wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
5317         ixl_flush(hw);
5318 }
5319
5320 static void
5321 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
5322 {
5323
5324         wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
5325             I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
5326         ixl_flush(hw);
5327 }
5328
5329 static void
5330 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
5331 {
5332         struct i40e_hw *hw;
5333         uint32_t vfint_reg, vpint_reg;
5334         int i;
5335
5336         hw = &pf->hw;
5337
5338         ixl_vf_vsi_release(pf, &vf->vsi);
5339
5340         /* Index 0 has a special register. */
5341         ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
5342
5343         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5344                 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
5345                 ixl_vf_disable_queue_intr(hw, vfint_reg);
5346         }
5347
5348         /* Index 0 has a special register. */
5349         ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
5350
5351         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5352                 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
5353                 ixl_vf_unregister_intr(hw, vpint_reg);
5354         }
5355
5356         vf->vsi.num_queues = 0;
5357 }
5358
5359 static int
5360 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
5361 {
5362         struct i40e_hw *hw;
5363         int i;
5364         uint16_t global_vf_num;
5365         uint32_t ciad;
5366
5367         hw = &pf->hw;
5368         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5369
5370         wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
5371              (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
5372         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5373                 ciad = rd32(hw, I40E_PF_PCI_CIAD);
5374                 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
5375                         return (0);
5376                 DELAY(1);
5377         }
5378
5379         return (ETIMEDOUT);
5380 }
5381
5382 static void
5383 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5384 {
5385         struct i40e_hw *hw;
5386         uint32_t vfrtrig;
5387
5388         hw = &pf->hw;
5389
5390         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5391         vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5392         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5393         ixl_flush(hw);
5394
5395         ixl_reinit_vf(pf, vf);
5396 }
5397
5398 static void
5399 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5400 {
5401         struct i40e_hw *hw;
5402         uint32_t vfrstat, vfrtrig;
5403         int i, error;
5404
5405         hw = &pf->hw;
5406
5407         error = ixl_flush_pcie(pf, vf);
5408         if (error != 0)
5409                 device_printf(pf->dev,
5410                     "Timed out waiting for PCIe activity to stop on VF-%d\n",
5411                     vf->vf_num);
5412
5413         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5414                 DELAY(10);
5415
5416                 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
5417                 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
5418                         break;
5419         }
5420
5421         if (i == IXL_VF_RESET_TIMEOUT)
5422                 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
5423
5424         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
5425
5426         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5427         vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5428         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5429
5430         if (vf->vsi.seid != 0)
5431                 ixl_disable_rings(&vf->vsi);
5432
5433         ixl_vf_release_resources(pf, vf);
5434         ixl_vf_setup_vsi(pf, vf);
5435         ixl_vf_map_queues(pf, vf);
5436
5437         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
5438         ixl_flush(hw);
5439 }
5440
5441 static const char *
5442 ixl_vc_opcode_str(uint16_t op)
5443 {
5444
5445         switch (op) {
5446         case I40E_VIRTCHNL_OP_VERSION:
5447                 return ("VERSION");
5448         case I40E_VIRTCHNL_OP_RESET_VF:
5449                 return ("RESET_VF");
5450         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
5451                 return ("GET_VF_RESOURCES");
5452         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
5453                 return ("CONFIG_TX_QUEUE");
5454         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
5455                 return ("CONFIG_RX_QUEUE");
5456         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
5457                 return ("CONFIG_VSI_QUEUES");
5458         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
5459                 return ("CONFIG_IRQ_MAP");
5460         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
5461                 return ("ENABLE_QUEUES");
5462         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
5463                 return ("DISABLE_QUEUES");
5464         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
5465                 return ("ADD_ETHER_ADDRESS");
5466         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
5467                 return ("DEL_ETHER_ADDRESS");
5468         case I40E_VIRTCHNL_OP_ADD_VLAN:
5469                 return ("ADD_VLAN");
5470         case I40E_VIRTCHNL_OP_DEL_VLAN:
5471                 return ("DEL_VLAN");
5472         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
5473                 return ("CONFIG_PROMISCUOUS_MODE");
5474         case I40E_VIRTCHNL_OP_GET_STATS:
5475                 return ("GET_STATS");
5476         case I40E_VIRTCHNL_OP_FCOE:
5477                 return ("FCOE");
5478         case I40E_VIRTCHNL_OP_EVENT:
5479                 return ("EVENT");
5480         default:
5481                 return ("UNKNOWN");
5482         }
5483 }
5484
5485 static int
5486 ixl_vc_opcode_level(uint16_t opcode)
5487 {
5488
5489         switch (opcode) {
5490         case I40E_VIRTCHNL_OP_GET_STATS:
5491                 return (10);
5492         default:
5493                 return (5);
5494         }
5495 }
5496
5497 static void
5498 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5499     enum i40e_status_code status, void *msg, uint16_t len)
5500 {
5501         struct i40e_hw *hw;
5502         int global_vf_id;
5503
5504         hw = &pf->hw;
5505         global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
5506
5507         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
5508             "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
5509             ixl_vc_opcode_str(op), op, status, vf->vf_num);
5510
5511         i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
5512 }
5513
5514 static void
5515 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
5516 {
5517
5518         ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
5519 }
5520
5521 static void
5522 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5523     enum i40e_status_code status, const char *file, int line)
5524 {
5525
5526         I40E_VC_DEBUG(pf, 1,
5527             "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
5528             ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
5529         ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
5530 }
5531
5532 static void
5533 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5534     uint16_t msg_size)
5535 {
5536         struct i40e_virtchnl_version_info reply;
5537
5538         if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
5539                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
5540                     I40E_ERR_PARAM);
5541                 return;
5542         }
5543
5544         reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
5545         reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
5546         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
5547             sizeof(reply));
5548 }
5549
5550 static void
5551 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5552     uint16_t msg_size)
5553 {
5554
5555         if (msg_size != 0) {
5556                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
5557                     I40E_ERR_PARAM);
5558                 return;
5559         }
5560
5561         ixl_reset_vf(pf, vf);
5562
5563         /* No response to a reset message. */
5564 }
5565
5566 static void
5567 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5568     uint16_t msg_size)
5569 {
5570         struct i40e_virtchnl_vf_resource reply;
5571
5572         if (msg_size != 0) {
5573                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5574                     I40E_ERR_PARAM);
5575                 return;
5576         }
5577
5578         bzero(&reply, sizeof(reply));
5579
5580         reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2;
5581
5582         reply.num_vsis = 1;
5583         reply.num_queue_pairs = vf->vsi.num_queues;
5584         reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
5585         reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
5586         reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
5587         reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
5588         memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
5589
5590         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5591             I40E_SUCCESS, &reply, sizeof(reply));
5592 }
5593
5594 static int
5595 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5596     struct i40e_virtchnl_txq_info *info)
5597 {
5598         struct i40e_hw *hw;
5599         struct i40e_hmc_obj_txq txq;
5600         uint16_t global_queue_num, global_vf_num;
5601         enum i40e_status_code status;
5602         uint32_t qtx_ctl;
5603
5604         hw = &pf->hw;
5605         global_queue_num = vf->vsi.first_queue + info->queue_id;
5606         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5607         bzero(&txq, sizeof(txq));
5608
5609         status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
5610         if (status != I40E_SUCCESS)
5611                 return (EINVAL);
5612
5613         txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
5614
5615         txq.head_wb_ena = info->headwb_enabled;
5616         txq.head_wb_addr = info->dma_headwb_addr;
5617         txq.qlen = info->ring_len;
5618         txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
5619         txq.rdylist_act = 0;
5620
5621         status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
5622         if (status != I40E_SUCCESS)
5623                 return (EINVAL);
5624
5625         qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
5626             (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
5627             (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
5628         wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
5629         ixl_flush(hw);
5630
5631         return (0);
5632 }
5633
5634 static int
5635 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5636     struct i40e_virtchnl_rxq_info *info)
5637 {
5638         struct i40e_hw *hw;
5639         struct i40e_hmc_obj_rxq rxq;
5640         uint16_t global_queue_num;
5641         enum i40e_status_code status;
5642
5643         hw = &pf->hw;
5644         global_queue_num = vf->vsi.first_queue + info->queue_id;
5645         bzero(&rxq, sizeof(rxq));
5646
5647         if (info->databuffer_size > IXL_VF_MAX_BUFFER)
5648                 return (EINVAL);
5649
5650         if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
5651             info->max_pkt_size < ETHER_MIN_LEN)
5652                 return (EINVAL);
5653
5654         if (info->splithdr_enabled) {
5655                 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
5656                         return (EINVAL);
5657
5658                 rxq.hsplit_0 = info->rx_split_pos &
5659                     (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
5660                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
5661                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
5662                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
5663                 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
5664
5665                 rxq.dtype = 2;
5666         }
5667
5668         status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
5669         if (status != I40E_SUCCESS)
5670                 return (EINVAL);
5671
5672         rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
5673         rxq.qlen = info->ring_len;
5674
5675         rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
5676
5677         rxq.dsize = 1;
5678         rxq.crcstrip = 1;
5679         rxq.l2tsel = 1;
5680
5681         rxq.rxmax = info->max_pkt_size;
5682         rxq.tphrdesc_ena = 1;
5683         rxq.tphwdesc_ena = 1;
5684         rxq.tphdata_ena = 1;
5685         rxq.tphhead_ena = 1;
5686         rxq.lrxqthresh = 2;
5687         rxq.prefena = 1;
5688
5689         status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
5690         if (status != I40E_SUCCESS)
5691                 return (EINVAL);
5692
5693         return (0);
5694 }
5695
5696 static void
5697 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5698     uint16_t msg_size)
5699 {
5700         struct i40e_virtchnl_vsi_queue_config_info *info;
5701         struct i40e_virtchnl_queue_pair_info *pair;
5702         int i;
5703
5704         if (msg_size < sizeof(*info)) {
5705                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5706                     I40E_ERR_PARAM);
5707                 return;
5708         }
5709
5710         info = msg;
5711         if (info->num_queue_pairs == 0) {
5712                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5713                     I40E_ERR_PARAM);
5714                 return;
5715         }
5716
5717         if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
5718                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5719                     I40E_ERR_PARAM);
5720                 return;
5721         }
5722
5723         if (info->vsi_id != vf->vsi.vsi_num) {
5724                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5725                     I40E_ERR_PARAM);
5726                 return;
5727         }
5728
5729         for (i = 0; i < info->num_queue_pairs; i++) {
5730                 pair = &info->qpair[i];
5731
5732                 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
5733                     pair->rxq.vsi_id != vf->vsi.vsi_num ||
5734                     pair->txq.queue_id != pair->rxq.queue_id ||
5735                     pair->txq.queue_id >= vf->vsi.num_queues) {
5736
5737                         i40e_send_vf_nack(pf, vf,
5738                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5739                         return;
5740                 }
5741
5742                 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
5743                         i40e_send_vf_nack(pf, vf,
5744                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5745                         return;
5746                 }
5747
5748                 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
5749                         i40e_send_vf_nack(pf, vf,
5750                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5751                         return;
5752                 }
5753         }
5754
5755         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
5756 }
5757
5758 static void
5759 ixl_vf_set_qctl(struct ixl_pf *pf,
5760     const struct i40e_virtchnl_vector_map *vector,
5761     enum i40e_queue_type cur_type, uint16_t cur_queue,
5762     enum i40e_queue_type *last_type, uint16_t *last_queue)
5763 {
5764         uint32_t offset, qctl;
5765         uint16_t itr_indx;
5766
5767         if (cur_type == I40E_QUEUE_TYPE_RX) {
5768                 offset = I40E_QINT_RQCTL(cur_queue);
5769                 itr_indx = vector->rxitr_idx;
5770         } else {
5771                 offset = I40E_QINT_TQCTL(cur_queue);
5772                 itr_indx = vector->txitr_idx;
5773         }
5774
5775         qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
5776             (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
5777             (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
5778             I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5779             (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
5780
5781         wr32(&pf->hw, offset, qctl);
5782
5783         *last_type = cur_type;
5784         *last_queue = cur_queue;
5785 }
5786
5787 static void
5788 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
5789     const struct i40e_virtchnl_vector_map *vector)
5790 {
5791         struct i40e_hw *hw;
5792         u_int qindex;
5793         enum i40e_queue_type type, last_type;
5794         uint32_t lnklst_reg;
5795         uint16_t rxq_map, txq_map, cur_queue, last_queue;
5796
5797         hw = &pf->hw;
5798
5799         rxq_map = vector->rxq_map;
5800         txq_map = vector->txq_map;
5801
5802         last_queue = IXL_END_OF_INTR_LNKLST;
5803         last_type = I40E_QUEUE_TYPE_RX;
5804
5805         /*
5806          * The datasheet says to optimize performance, RX queues and TX queues
5807          * should be interleaved in the interrupt linked list, so we process
5808          * both at once here.
5809          */
5810         while ((rxq_map != 0) || (txq_map != 0)) {
5811                 if (txq_map != 0) {
5812                         qindex = ffs(txq_map) - 1;
5813                         type = I40E_QUEUE_TYPE_TX;
5814                         cur_queue = vf->vsi.first_queue + qindex;
5815                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
5816                             &last_type, &last_queue);
5817                         txq_map &= ~(1 << qindex);
5818                 }
5819
5820                 if (rxq_map != 0) {
5821                         qindex = ffs(rxq_map) - 1;
5822                         type = I40E_QUEUE_TYPE_RX;
5823                         cur_queue = vf->vsi.first_queue + qindex;
5824                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
5825                             &last_type, &last_queue);
5826                         rxq_map &= ~(1 << qindex);
5827                 }
5828         }
5829
5830         if (vector->vector_id == 0)
5831                 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
5832         else
5833                 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
5834                     vf->vf_num);
5835         wr32(hw, lnklst_reg,
5836             (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
5837             (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
5838
5839         ixl_flush(hw);
5840 }
5841
5842 static void
5843 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5844     uint16_t msg_size)
5845 {
5846         struct i40e_virtchnl_irq_map_info *map;
5847         struct i40e_virtchnl_vector_map *vector;
5848         struct i40e_hw *hw;
5849         int i, largest_txq, largest_rxq;
5850
5851         hw = &pf->hw;
5852
5853         if (msg_size < sizeof(*map)) {
5854                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5855                     I40E_ERR_PARAM);
5856                 return;
5857         }
5858
5859         map = msg;
5860         if (map->num_vectors == 0) {
5861                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5862                     I40E_ERR_PARAM);
5863                 return;
5864         }
5865
5866         if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
5867                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5868                     I40E_ERR_PARAM);
5869                 return;
5870         }
5871
5872         for (i = 0; i < map->num_vectors; i++) {
5873                 vector = &map->vecmap[i];
5874
5875                 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
5876                     vector->vsi_id != vf->vsi.vsi_num) {
5877                         i40e_send_vf_nack(pf, vf,
5878                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
5879                         return;
5880                 }
5881
5882                 if (vector->rxq_map != 0) {
5883                         largest_rxq = fls(vector->rxq_map) - 1;
5884                         if (largest_rxq >= vf->vsi.num_queues) {
5885                                 i40e_send_vf_nack(pf, vf,
5886                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5887                                     I40E_ERR_PARAM);
5888                                 return;
5889                         }
5890                 }
5891
5892                 if (vector->txq_map != 0) {
5893                         largest_txq = fls(vector->txq_map) - 1;
5894                         if (largest_txq >= vf->vsi.num_queues) {
5895                                 i40e_send_vf_nack(pf, vf,
5896                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5897                                     I40E_ERR_PARAM);
5898                                 return;
5899                         }
5900                 }
5901
5902                 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
5903                     vector->txitr_idx > IXL_MAX_ITR_IDX) {
5904                         i40e_send_vf_nack(pf, vf,
5905                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5906                             I40E_ERR_PARAM);
5907                         return;
5908                 }
5909
5910                 ixl_vf_config_vector(pf, vf, vector);
5911         }
5912
5913         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
5914 }
5915
5916 static void
5917 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5918     uint16_t msg_size)
5919 {
5920         struct i40e_virtchnl_queue_select *select;
5921         int error;
5922
5923         if (msg_size != sizeof(*select)) {
5924                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5925                     I40E_ERR_PARAM);
5926                 return;
5927         }
5928
5929         select = msg;
5930         if (select->vsi_id != vf->vsi.vsi_num ||
5931             select->rx_queues == 0 || select->tx_queues == 0) {
5932                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5933                     I40E_ERR_PARAM);
5934                 return;
5935         }
5936
5937         error = ixl_enable_rings(&vf->vsi);
5938         if (error) {
5939                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5940                     I40E_ERR_TIMEOUT);
5941                 return;
5942         }
5943
5944         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
5945 }
5946
5947 static void
5948 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
5949     void *msg, uint16_t msg_size)
5950 {
5951         struct i40e_virtchnl_queue_select *select;
5952         int error;
5953
5954         if (msg_size != sizeof(*select)) {
5955                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5956                     I40E_ERR_PARAM);
5957                 return;
5958         }
5959
5960         select = msg;
5961         if (select->vsi_id != vf->vsi.vsi_num ||
5962             select->rx_queues == 0 || select->tx_queues == 0) {
5963                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5964                     I40E_ERR_PARAM);
5965                 return;
5966         }
5967
5968         error = ixl_disable_rings(&vf->vsi);
5969         if (error) {
5970                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5971                     I40E_ERR_TIMEOUT);
5972                 return;
5973         }
5974
5975         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
5976 }
5977
5978 static boolean_t
5979 ixl_zero_mac(const uint8_t *addr)
5980 {
5981         uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
5982
5983         return (cmp_etheraddr(addr, zero));
5984 }
5985
5986 static boolean_t
5987 ixl_bcast_mac(const uint8_t *addr)
5988 {
5989
5990         return (cmp_etheraddr(addr, ixl_bcast_addr));
5991 }
5992
5993 static int
5994 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
5995 {
5996
5997         if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
5998                 return (EINVAL);
5999
6000         /*
6001          * If the VF is not allowed to change its MAC address, don't let it
6002          * set a MAC filter for an address that is not a multicast address and
6003          * is not its assigned MAC.
6004          */
6005         if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6006             !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6007                 return (EPERM);
6008
6009         return (0);
6010 }
6011
6012 static void
6013 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6014     uint16_t msg_size)
6015 {
6016         struct i40e_virtchnl_ether_addr_list *addr_list;
6017         struct i40e_virtchnl_ether_addr *addr;
6018         struct ixl_vsi *vsi;
6019         int i;
6020         size_t expected_size;
6021
6022         vsi = &vf->vsi;
6023
6024         if (msg_size < sizeof(*addr_list)) {
6025                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6026                     I40E_ERR_PARAM);
6027                 return;
6028         }
6029
6030         addr_list = msg;
6031         expected_size = sizeof(*addr_list) +
6032             addr_list->num_elements * sizeof(*addr);
6033
6034         if (addr_list->num_elements == 0 ||
6035             addr_list->vsi_id != vsi->vsi_num ||
6036             msg_size != expected_size) {
6037                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6038                     I40E_ERR_PARAM);
6039                 return;
6040         }
6041
6042         for (i = 0; i < addr_list->num_elements; i++) {
6043                 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6044                         i40e_send_vf_nack(pf, vf,
6045                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6046                         return;
6047                 }
6048         }
6049
6050         for (i = 0; i < addr_list->num_elements; i++) {
6051                 addr = &addr_list->list[i];
6052                 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6053         }
6054
6055         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6056 }
6057
6058 static void
6059 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6060     uint16_t msg_size)
6061 {
6062         struct i40e_virtchnl_ether_addr_list *addr_list;
6063         struct i40e_virtchnl_ether_addr *addr;
6064         size_t expected_size;
6065         int i;
6066
6067         if (msg_size < sizeof(*addr_list)) {
6068                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6069                     I40E_ERR_PARAM);
6070                 return;
6071         }
6072
6073         addr_list = msg;
6074         expected_size = sizeof(*addr_list) +
6075             addr_list->num_elements * sizeof(*addr);
6076
6077         if (addr_list->num_elements == 0 ||
6078             addr_list->vsi_id != vf->vsi.vsi_num ||
6079             msg_size != expected_size) {
6080                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6081                     I40E_ERR_PARAM);
6082                 return;
6083         }
6084
6085         for (i = 0; i < addr_list->num_elements; i++) {
6086                 addr = &addr_list->list[i];
6087                 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6088                         i40e_send_vf_nack(pf, vf,
6089                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6090                         return;
6091                 }
6092         }
6093
6094         for (i = 0; i < addr_list->num_elements; i++) {
6095                 addr = &addr_list->list[i];
6096                 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6097         }
6098
6099         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6100 }
6101
6102 static enum i40e_status_code
6103 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6104 {
6105         struct i40e_vsi_context vsi_ctx;
6106
6107         vsi_ctx.seid = vf->vsi.seid;
6108
6109         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6110         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6111         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6112             I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6113         return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6114 }
6115
6116 static void
6117 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6118     uint16_t msg_size)
6119 {
6120         struct i40e_virtchnl_vlan_filter_list *filter_list;
6121         enum i40e_status_code code;
6122         size_t expected_size;
6123         int i;
6124
6125         if (msg_size < sizeof(*filter_list)) {
6126                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6127                     I40E_ERR_PARAM);
6128                 return;
6129         }
6130
6131         filter_list = msg;
6132         expected_size = sizeof(*filter_list) +
6133             filter_list->num_elements * sizeof(uint16_t);
6134         if (filter_list->num_elements == 0 ||
6135             filter_list->vsi_id != vf->vsi.vsi_num ||
6136             msg_size != expected_size) {
6137                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6138                     I40E_ERR_PARAM);
6139                 return;
6140         }
6141
6142         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6143                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6144                     I40E_ERR_PARAM);
6145                 return;
6146         }
6147
6148         for (i = 0; i < filter_list->num_elements; i++) {
6149                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6150                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6151                             I40E_ERR_PARAM);
6152                         return;
6153                 }
6154         }
6155
6156         code = ixl_vf_enable_vlan_strip(pf, vf);
6157         if (code != I40E_SUCCESS) {
6158                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6159                     I40E_ERR_PARAM);
6160         }
6161
6162         for (i = 0; i < filter_list->num_elements; i++)
6163                 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6164
6165         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
6166 }
6167
6168 static void
6169 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6170     uint16_t msg_size)
6171 {
6172         struct i40e_virtchnl_vlan_filter_list *filter_list;
6173         int i;
6174         size_t expected_size;
6175
6176         if (msg_size < sizeof(*filter_list)) {
6177                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6178                     I40E_ERR_PARAM);
6179                 return;
6180         }
6181
6182         filter_list = msg;
6183         expected_size = sizeof(*filter_list) +
6184             filter_list->num_elements * sizeof(uint16_t);
6185         if (filter_list->num_elements == 0 ||
6186             filter_list->vsi_id != vf->vsi.vsi_num ||
6187             msg_size != expected_size) {
6188                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6189                     I40E_ERR_PARAM);
6190                 return;
6191         }
6192
6193         for (i = 0; i < filter_list->num_elements; i++) {
6194                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6195                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6196                             I40E_ERR_PARAM);
6197                         return;
6198                 }
6199         }
6200
6201         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6202                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6203                     I40E_ERR_PARAM);
6204                 return;
6205         }
6206
6207         for (i = 0; i < filter_list->num_elements; i++)
6208                 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6209
6210         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
6211 }
6212
6213 static void
6214 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6215     void *msg, uint16_t msg_size)
6216 {
6217         struct i40e_virtchnl_promisc_info *info;
6218         enum i40e_status_code code;
6219
6220         if (msg_size != sizeof(*info)) {
6221                 i40e_send_vf_nack(pf, vf,
6222                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6223                 return;
6224         }
6225
6226         if (!vf->vf_flags & VF_FLAG_PROMISC_CAP) {
6227                 i40e_send_vf_nack(pf, vf,
6228                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6229                 return;
6230         }
6231
6232         info = msg;
6233         if (info->vsi_id != vf->vsi.vsi_num) {
6234                 i40e_send_vf_nack(pf, vf,
6235                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6236                 return;
6237         }
6238
6239         code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
6240             info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
6241         if (code != I40E_SUCCESS) {
6242                 i40e_send_vf_nack(pf, vf,
6243                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6244                 return;
6245         }
6246
6247         code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
6248             info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
6249         if (code != I40E_SUCCESS) {
6250                 i40e_send_vf_nack(pf, vf,
6251                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6252                 return;
6253         }
6254
6255         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
6256 }
6257
6258 static void
6259 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6260     uint16_t msg_size)
6261 {
6262         struct i40e_virtchnl_queue_select *queue;
6263
6264         if (msg_size != sizeof(*queue)) {
6265                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6266                     I40E_ERR_PARAM);
6267                 return;
6268         }
6269
6270         queue = msg;
6271         if (queue->vsi_id != vf->vsi.vsi_num) {
6272                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6273                     I40E_ERR_PARAM);
6274                 return;
6275         }
6276
6277         ixl_update_eth_stats(&vf->vsi);
6278
6279         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6280             I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
6281 }
6282
6283 static void
6284 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
6285 {
6286         struct ixl_vf *vf;
6287         void *msg;
6288         uint16_t vf_num, msg_size;
6289         uint32_t opcode;
6290
6291         vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
6292         opcode = le32toh(event->desc.cookie_high);
6293
6294         if (vf_num >= pf->num_vfs) {
6295                 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
6296                 return;
6297         }
6298
6299         vf = &pf->vfs[vf_num];
6300         msg = event->msg_buf;
6301         msg_size = event->msg_len;
6302
6303         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
6304             "Got msg %s(%d) from VF-%d of size %d\n",
6305             ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
6306
6307         switch (opcode) {
6308         case I40E_VIRTCHNL_OP_VERSION:
6309                 ixl_vf_version_msg(pf, vf, msg, msg_size);
6310                 break;
6311         case I40E_VIRTCHNL_OP_RESET_VF:
6312                 ixl_vf_reset_msg(pf, vf, msg, msg_size);
6313                 break;
6314         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6315                 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
6316                 break;
6317         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6318                 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
6319                 break;
6320         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6321                 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
6322                 break;
6323         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6324                 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
6325                 break;
6326         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6327                 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
6328                 break;
6329         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6330                 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
6331                 break;
6332         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6333                 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
6334                 break;
6335         case I40E_VIRTCHNL_OP_ADD_VLAN:
6336                 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
6337                 break;
6338         case I40E_VIRTCHNL_OP_DEL_VLAN:
6339                 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
6340                 break;
6341         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6342                 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
6343                 break;
6344         case I40E_VIRTCHNL_OP_GET_STATS:
6345                 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
6346                 break;
6347
6348         /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
6349         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6350         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6351         default:
6352                 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
6353                 break;
6354         }
6355 }
6356
6357 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
6358 static void
6359 ixl_handle_vflr(void *arg, int pending)
6360 {
6361         struct ixl_pf *pf;
6362         struct i40e_hw *hw;
6363         uint16_t global_vf_num;
6364         uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
6365         int i;
6366
6367         pf = arg;
6368         hw = &pf->hw;
6369
6370         IXL_PF_LOCK(pf);
6371         for (i = 0; i < pf->num_vfs; i++) {
6372                 global_vf_num = hw->func_caps.vf_base_id + i;
6373
6374                 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
6375                 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
6376                 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
6377                 if (vflrstat & vflrstat_mask) {
6378                         wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
6379                             vflrstat_mask);
6380
6381                         ixl_reinit_vf(pf, &pf->vfs[i]);
6382                 }
6383         }
6384
6385         icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
6386         icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
6387         wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
6388         ixl_flush(hw);
6389
6390         IXL_PF_UNLOCK(pf);
6391 }
6392
6393 static int
6394 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
6395 {
6396
6397         switch (err) {
6398         case I40E_AQ_RC_EPERM:
6399                 return (EPERM);
6400         case I40E_AQ_RC_ENOENT:
6401                 return (ENOENT);
6402         case I40E_AQ_RC_ESRCH:
6403                 return (ESRCH);
6404         case I40E_AQ_RC_EINTR:
6405                 return (EINTR);
6406         case I40E_AQ_RC_EIO:
6407                 return (EIO);
6408         case I40E_AQ_RC_ENXIO:
6409                 return (ENXIO);
6410         case I40E_AQ_RC_E2BIG:
6411                 return (E2BIG);
6412         case I40E_AQ_RC_EAGAIN:
6413                 return (EAGAIN);
6414         case I40E_AQ_RC_ENOMEM:
6415                 return (ENOMEM);
6416         case I40E_AQ_RC_EACCES:
6417                 return (EACCES);
6418         case I40E_AQ_RC_EFAULT:
6419                 return (EFAULT);
6420         case I40E_AQ_RC_EBUSY:
6421                 return (EBUSY);
6422         case I40E_AQ_RC_EEXIST:
6423                 return (EEXIST);
6424         case I40E_AQ_RC_EINVAL:
6425                 return (EINVAL);
6426         case I40E_AQ_RC_ENOTTY:
6427                 return (ENOTTY);
6428         case I40E_AQ_RC_ENOSPC:
6429                 return (ENOSPC);
6430         case I40E_AQ_RC_ENOSYS:
6431                 return (ENOSYS);
6432         case I40E_AQ_RC_ERANGE:
6433                 return (ERANGE);
6434         case I40E_AQ_RC_EFLUSHED:
6435                 return (EINVAL);        /* No exact equivalent in errno.h */
6436         case I40E_AQ_RC_BAD_ADDR:
6437                 return (EFAULT);
6438         case I40E_AQ_RC_EMODE:
6439                 return (EPERM);
6440         case I40E_AQ_RC_EFBIG:
6441                 return (EFBIG);
6442         default:
6443                 return (EINVAL);
6444         }
6445 }
6446
6447 static int
6448 ixl_init_iov(device_t dev, uint16_t num_vfs, const nvlist_t *params)
6449 {
6450         struct ixl_pf *pf;
6451         struct i40e_hw *hw;
6452         struct ixl_vsi *pf_vsi;
6453         enum i40e_status_code ret;
6454         int i, error;
6455
6456         pf = device_get_softc(dev);
6457         hw = &pf->hw;
6458         pf_vsi = &pf->vsi;
6459
6460         IXL_PF_LOCK(pf);
6461         pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
6462             M_ZERO);
6463
6464         if (pf->vfs == NULL) {
6465                 error = ENOMEM;
6466                 goto fail;
6467         }
6468
6469         for (i = 0; i < num_vfs; i++)
6470                 sysctl_ctx_init(&pf->vfs[i].ctx);
6471
6472         ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
6473             1, FALSE, FALSE, &pf->veb_seid, NULL);
6474         if (ret != I40E_SUCCESS) {
6475                 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
6476                 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
6477                     error);
6478                 goto fail;
6479         }
6480
6481         ixl_configure_msix(pf);
6482         ixl_enable_adminq(hw);
6483
6484         pf->num_vfs = num_vfs;
6485         IXL_PF_UNLOCK(pf);
6486         return (0);
6487
6488 fail:
6489         free(pf->vfs, M_IXL);
6490         pf->vfs = NULL;
6491         IXL_PF_UNLOCK(pf);
6492         return (error);
6493 }
6494
6495 static void
6496 ixl_uninit_iov(device_t dev)
6497 {
6498         struct ixl_pf *pf;
6499         struct i40e_hw *hw;
6500         struct ixl_vsi *vsi;
6501         struct ifnet *ifp;
6502         struct ixl_vf *vfs;
6503         int i, num_vfs;
6504
6505         pf = device_get_softc(dev);
6506         hw = &pf->hw;
6507         vsi = &pf->vsi;
6508         ifp = vsi->ifp;
6509
6510         IXL_PF_LOCK(pf);
6511         for (i = 0; i < pf->num_vfs; i++) {
6512                 if (pf->vfs[i].vsi.seid != 0)
6513                         i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
6514         }
6515
6516         if (pf->veb_seid != 0) {
6517                 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
6518                 pf->veb_seid = 0;
6519         }
6520
6521         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
6522                 ixl_disable_intr(vsi);
6523
6524         vfs = pf->vfs;
6525         num_vfs = pf->num_vfs;
6526
6527         pf->vfs = NULL;
6528         pf->num_vfs = 0;
6529         IXL_PF_UNLOCK(pf);
6530
6531         /* Do this after the unlock as sysctl_ctx_free might sleep. */
6532         for (i = 0; i < num_vfs; i++)
6533                 sysctl_ctx_free(&vfs[i].ctx);
6534         free(vfs, M_IXL);
6535 }
6536
6537 static int
6538 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
6539 {
6540         char sysctl_name[QUEUE_NAME_LEN];
6541         struct ixl_pf *pf;
6542         struct ixl_vf *vf;
6543         const void *mac;
6544         size_t size;
6545         int error;
6546
6547         pf = device_get_softc(dev);
6548         vf = &pf->vfs[vfnum];
6549
6550         IXL_PF_LOCK(pf);
6551         vf->vf_num = vfnum;
6552
6553         vf->vsi.back = pf;
6554         vf->vf_flags = VF_FLAG_ENABLED;
6555         SLIST_INIT(&vf->vsi.ftl);
6556
6557         error = ixl_vf_setup_vsi(pf, vf);
6558         if (error != 0)
6559                 goto out;
6560
6561         if (nvlist_exists_binary(params, "mac-addr")) {
6562                 mac = nvlist_get_binary(params, "mac-addr", &size);
6563                 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
6564
6565                 if (nvlist_get_bool(params, "allow-set-mac"))
6566                         vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6567         } else
6568                 /*
6569                  * If the administrator has not specified a MAC address then
6570                  * we must allow the VF to choose one.
6571                  */
6572                 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6573
6574         if (nvlist_get_bool(params, "mac-anti-spoof"))
6575                 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
6576
6577         if (nvlist_get_bool(params, "allow-promisc"))
6578                 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
6579
6580         vf->vf_flags |= VF_FLAG_VLAN_CAP;
6581
6582         ixl_reset_vf(pf, vf);
6583 out:
6584         IXL_PF_UNLOCK(pf);
6585         if (error == 0) {
6586                 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
6587                 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
6588         }
6589
6590         return (error);
6591 }
6592 #endif /* PCI_IOV */