sys/dev/ixl/if_ixl.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifndef IXL_STANDALONE_BUILD
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "opt_rss.h"
  39 #endif
  40
  41 #include "ixl.h"
  42 #include "ixl_pf.h"
  43
  44 #ifdef RSS
  45 #include <net/rss_config.h>
  46 #endif
  47
  48 /*********************************************************************
  49  *  Driver version
  50  *********************************************************************/
  51 char ixl_driver_version[] = "1.4.1";
  52
  53 /*********************************************************************
  54  *  PCI Device ID Table
  55  *
  56  *  Used by probe to select devices to load on
  57  *  Last field stores an index into ixl_strings
  58  *  Last entry must be all 0s
  59  *
  60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  61  *********************************************************************/
  62
  63 static ixl_vendor_info_t ixl_vendor_info_array[] =
  64 {
  65         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
  66         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_A, 0, 0, 0},
  67         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
  68         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
  69         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
  70         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
  71         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
  72         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
  73         {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2, 0, 0, 0},
  74         /* required last entry */
  75         {0, 0, 0, 0, 0}
  76 };
  77
  78 /*********************************************************************
  79  *  Table of branding strings
  80  *********************************************************************/
  81
  82 static char    *ixl_strings[] = {
  83         "Intel(R) Ethernet Connection XL710 Driver"
  84 };
  85
  86
  87 /*********************************************************************
  88  *  Function prototypes
  89  *********************************************************************/
  90 static int      ixl_probe(device_t);
  91 static int      ixl_attach(device_t);
  92 static int      ixl_detach(device_t);
  93 static int      ixl_shutdown(device_t);
  94 static int      ixl_get_hw_capabilities(struct ixl_pf *);
  95 static void     ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
  96 static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
  97 static void     ixl_init(void *);
  98 static void     ixl_init_locked(struct ixl_pf *);
  99 static void     ixl_stop(struct ixl_pf *);
 100 static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
 101 static int      ixl_media_change(struct ifnet *);
 102 static void     ixl_update_link_status(struct ixl_pf *);
 103 static int      ixl_allocate_pci_resources(struct ixl_pf *);
 104 static u16      ixl_get_bus_info(struct i40e_hw *, device_t);
 105 static int      ixl_setup_stations(struct ixl_pf *);
 106 static int      ixl_switch_config(struct ixl_pf *);
 107 static int      ixl_initialize_vsi(struct ixl_vsi *);
 108 static int      ixl_assign_vsi_msix(struct ixl_pf *);
 109 static int      ixl_assign_vsi_legacy(struct ixl_pf *);
 110 static int      ixl_init_msix(struct ixl_pf *);
 111 static void     ixl_configure_msix(struct ixl_pf *);
 112 static void     ixl_configure_itr(struct ixl_pf *);
 113 static void     ixl_configure_legacy(struct ixl_pf *);
 114 static void     ixl_free_pci_resources(struct ixl_pf *);
 115 static void     ixl_local_timer(void *);
 116 static int      ixl_setup_interface(device_t, struct ixl_vsi *);
 117 static void     ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
 118 static void     ixl_config_rss(struct ixl_vsi *);
 119 static void     ixl_set_queue_rx_itr(struct ixl_queue *);
 120 static void     ixl_set_queue_tx_itr(struct ixl_queue *);
 121 static int      ixl_set_advertised_speeds(struct ixl_pf *, int);
 122
 123 static int      ixl_enable_rings(struct ixl_vsi *);
 124 static int      ixl_disable_rings(struct ixl_vsi *);
 125 static void     ixl_enable_intr(struct ixl_vsi *);
 126 static void     ixl_disable_intr(struct ixl_vsi *);
 127 static void     ixl_disable_rings_intr(struct ixl_vsi *);
 128
 129 static void     ixl_enable_adminq(struct i40e_hw *);
 130 static void     ixl_disable_adminq(struct i40e_hw *);
 131 static void     ixl_enable_queue(struct i40e_hw *, int);
 132 static void     ixl_disable_queue(struct i40e_hw *, int);
 133 static void     ixl_enable_legacy(struct i40e_hw *);
 134 static void     ixl_disable_legacy(struct i40e_hw *);
 135
 136 static void     ixl_set_promisc(struct ixl_vsi *);
 137 static void     ixl_add_multi(struct ixl_vsi *);
 138 static void     ixl_del_multi(struct ixl_vsi *);
 139 static void     ixl_register_vlan(void *, struct ifnet *, u16);
 140 static void     ixl_unregister_vlan(void *, struct ifnet *, u16);
 141 static void     ixl_setup_vlan_filters(struct ixl_vsi *);
 142
 143 static void     ixl_init_filters(struct ixl_vsi *);
 144 static void     ixl_reconfigure_filters(struct ixl_vsi *vsi);
 145 static void     ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
 146 static void     ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
 147 static void     ixl_add_hw_filters(struct ixl_vsi *, int, int);
 148 static void     ixl_del_hw_filters(struct ixl_vsi *, int);
 149 static struct ixl_mac_filter *
 150                 ixl_find_filter(struct ixl_vsi *, u8 *, s16);
 151 static void     ixl_add_mc_filter(struct ixl_vsi *, u8 *);
 152 static void     ixl_free_mac_filters(struct ixl_vsi *vsi);
 153
 154
 155 /* Sysctl debug interface */
 156 static int      ixl_debug_info(SYSCTL_HANDLER_ARGS);
 157 static void     ixl_print_debug_info(struct ixl_pf *);
 158
 159 /* The MSI/X Interrupt handlers */
 160 static void     ixl_intr(void *);
 161 static void     ixl_msix_que(void *);
 162 static void     ixl_msix_adminq(void *);
 163 static void     ixl_handle_mdd_event(struct ixl_pf *);
 164
 165 /* Deferred interrupt tasklets */
 166 static void     ixl_do_adminq(void *, int);
 167
 168 /* Sysctl handlers */
 169 static int      ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
 170 static int      ixl_set_advertise(SYSCTL_HANDLER_ARGS);
 171 static int      ixl_current_speed(SYSCTL_HANDLER_ARGS);
 172 static int      ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
 173
 174 /* Statistics */
 175 static void     ixl_add_hw_stats(struct ixl_pf *);
 176 static void     ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
 177                     struct sysctl_oid_list *, struct i40e_hw_port_stats *);
 178 static void     ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
 179                     struct sysctl_oid_list *,
 180                     struct i40e_eth_stats *);
 181 static void     ixl_update_stats_counters(struct ixl_pf *);
 182 static void     ixl_update_eth_stats(struct ixl_vsi *);
 183 static void     ixl_update_vsi_stats(struct ixl_vsi *);
 184 static void     ixl_pf_reset_stats(struct ixl_pf *);
 185 static void     ixl_vsi_reset_stats(struct ixl_vsi *);
 186 static void     ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
 187                     u64 *, u64 *);
 188 static void     ixl_stat_update32(struct i40e_hw *, u32, bool,
 189                     u64 *, u64 *);
 190
 191 #ifdef IXL_DEBUG_SYSCTL
 192 static int      ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
 193 static int      ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
 194 static int      ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 195 static int      ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
 196 static int      ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
 197 #endif
 198
 199 #ifdef PCI_IOV
 200 static int      ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
 201
 202 static int      ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
 203 static void     ixl_iov_uninit(device_t dev);
 204 static int      ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
 205
 206 static void     ixl_handle_vf_msg(struct ixl_pf *,
 207                     struct i40e_arq_event_info *);
 208 static void     ixl_handle_vflr(void *arg, int pending);
 209
 210 static void     ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 211 static void     ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
 212 #endif
 213
 214 /*********************************************************************
 215  *  FreeBSD Device Interface Entry Points
 216  *********************************************************************/
 217
 218 static device_method_t ixl_methods[] = {
 219         /* Device interface */
 220         DEVMETHOD(device_probe, ixl_probe),
 221         DEVMETHOD(device_attach, ixl_attach),
 222         DEVMETHOD(device_detach, ixl_detach),
 223         DEVMETHOD(device_shutdown, ixl_shutdown),
 224 #ifdef PCI_IOV
 225         DEVMETHOD(pci_iov_init, ixl_iov_init),
 226         DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
 227         DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
 228 #endif
 229         {0, 0}
 230 };
 231
 232 static driver_t ixl_driver = {
 233         "ixl", ixl_methods, sizeof(struct ixl_pf),
 234 };
 235
 236 devclass_t ixl_devclass;
 237 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
 238
 239 MODULE_DEPEND(ixl, pci, 1, 1, 1);
 240 MODULE_DEPEND(ixl, ether, 1, 1, 1);
 241
 242 #ifdef DEV_NETMAP
 243 MODULE_DEPEND(ixl, netmap, 1, 1, 1);
 244 #endif /* DEV_NETMAP */
 245
 246
 247 /*
 248 ** Global reset mutex
 249 */
 250 static struct mtx ixl_reset_mtx;
 251
 252 /*
 253 ** TUNEABLE PARAMETERS:
 254 */
 255
 256 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
 257                    "IXL driver parameters");
 258
 259 /*
 260  * MSIX should be the default for best performance,
 261  * but this allows it to be forced off for testing.
 262  */
 263 static int ixl_enable_msix = 1;
 264 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
 265 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
 266     "Enable MSI-X interrupts");
 267
 268 /*
 269 ** Number of descriptors per ring:
 270 **   - TX and RX are the same size
 271 */
 272 static int ixl_ringsz = DEFAULT_RING;
 273 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
 274 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
 275     &ixl_ringsz, 0, "Descriptor Ring Size");
 276
 277 /*
 278 ** This can be set manually, if left as 0 the
 279 ** number of queues will be calculated based
 280 ** on cpus and msix vectors available.
 281 */
 282 int ixl_max_queues = 0;
 283 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
 284 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
 285     &ixl_max_queues, 0, "Number of Queues");
 286
 287 /*
 288 ** Controls for Interrupt Throttling
 289 **      - true/false for dynamic adjustment
 290 **      - default values for static ITR
 291 */
 292 int ixl_dynamic_rx_itr = 0;
 293 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
 294 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
 295     &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
 296
 297 int ixl_dynamic_tx_itr = 0;
 298 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
 299 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
 300     &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
 301
 302 int ixl_rx_itr = IXL_ITR_8K;
 303 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
 304 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
 305     &ixl_rx_itr, 0, "RX Interrupt Rate");
 306
 307 int ixl_tx_itr = IXL_ITR_4K;
 308 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
 309 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
 310     &ixl_tx_itr, 0, "TX Interrupt Rate");
 311
 312 #ifdef IXL_FDIR
 313 static int ixl_enable_fdir = 1;
 314 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
 315 /* Rate at which we sample */
 316 int ixl_atr_rate = 20;
 317 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
 318 #endif
 319
 320 #ifdef DEV_NETMAP
 321 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
 322 #include <dev/netmap/if_ixl_netmap.h>
 323 #endif /* DEV_NETMAP */
 324
 325 static char *ixl_fc_string[6] = {
 326         "None",
 327         "Rx",
 328         "Tx",
 329         "Full",
 330         "Priority",
 331         "Default"
 332 };
 333
 334 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
 335
 336 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
 337     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 338
 339 /*********************************************************************
 340  *  Device identification routine
 341  *
 342  *  ixl_probe determines if the driver should be loaded on
 343  *  the hardware based on PCI vendor/device id of the device.
 344  *
 345  *  return BUS_PROBE_DEFAULT on success, positive on failure
 346  *********************************************************************/
 347
 348 static int
 349 ixl_probe(device_t dev)
 350 {
 351         ixl_vendor_info_t *ent;
 352
 353         u16     pci_vendor_id, pci_device_id;
 354         u16     pci_subvendor_id, pci_subdevice_id;
 355         char    device_name[256];
 356         static bool lock_init = FALSE;
 357
 358         INIT_DEBUGOUT("ixl_probe: begin");
 359
 360         pci_vendor_id = pci_get_vendor(dev);
 361         if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
 362                 return (ENXIO);
 363
 364         pci_device_id = pci_get_device(dev);
 365         pci_subvendor_id = pci_get_subvendor(dev);
 366         pci_subdevice_id = pci_get_subdevice(dev);
 367
 368         ent = ixl_vendor_info_array;
 369         while (ent->vendor_id != 0) {
 370                 if ((pci_vendor_id == ent->vendor_id) &&
 371                     (pci_device_id == ent->device_id) &&
 372
 373                     ((pci_subvendor_id == ent->subvendor_id) ||
 374                      (ent->subvendor_id == 0)) &&
 375
 376                     ((pci_subdevice_id == ent->subdevice_id) ||
 377                      (ent->subdevice_id == 0))) {
 378                         sprintf(device_name, "%s, Version - %s",
 379                                 ixl_strings[ent->index],
 380                                 ixl_driver_version);
 381                         device_set_desc_copy(dev, device_name);
 382                         /* One shot mutex init */
 383                         if (lock_init == FALSE) {
 384                                 lock_init = TRUE;
 385                                 mtx_init(&ixl_reset_mtx,
 386                                     "ixl_reset",
 387                                     "IXL RESET Lock", MTX_DEF);
 388                         }
 389                         return (BUS_PROBE_DEFAULT);
 390                 }
 391                 ent++;
 392         }
 393         return (ENXIO);
 394 }
 395
 396 /*********************************************************************
 397  *  Device initialization routine
 398  *
 399  *  The attach entry point is called when the driver is being loaded.
 400  *  This routine identifies the type of hardware, allocates all resources
 401  *  and initializes the hardware.
 402  *
 403  *  return 0 on success, positive on failure
 404  *********************************************************************/
 405
 406 static int
 407 ixl_attach(device_t dev)
 408 {
 409         struct ixl_pf   *pf;
 410         struct i40e_hw  *hw;
 411         struct ixl_vsi *vsi;
 412         u16             bus;
 413         int             error = 0;
 414 #ifdef PCI_IOV
 415         nvlist_t        *pf_schema, *vf_schema;
 416         int             iov_error;
 417 #endif
 418
 419         INIT_DEBUGOUT("ixl_attach: begin");
 420
 421         /* Allocate, clear, and link in our primary soft structure */
 422         pf = device_get_softc(dev);
 423         pf->dev = pf->osdep.dev = dev;
 424         hw = &pf->hw;
 425
 426         /*
 427         ** Note this assumes we have a single embedded VSI,
 428         ** this could be enhanced later to allocate multiple
 429         */
 430         vsi = &pf->vsi;
 431         vsi->dev = pf->dev;
 432
 433         /* Core Lock Init*/
 434         IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
 435
 436         /* Set up the timer callout */
 437         callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
 438
 439         /* Set up sysctls */
 440         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 441             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 442             OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 443             pf, 0, ixl_set_flowcntl, "I", "Flow Control");
 444
 445         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 446             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 447             OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 448             pf, 0, ixl_set_advertise, "I", "Advertised Speed");
 449
 450         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 451             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 452             OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
 453             pf, 0, ixl_current_speed, "A", "Current Port Speed");
 454
 455         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 456             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 457             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
 458             pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
 459
 460         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 461             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 462             OID_AUTO, "rx_itr", CTLFLAG_RW,
 463             &ixl_rx_itr, IXL_ITR_8K, "RX ITR");
 464
 465         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 466             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 467             OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
 468             &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
 469
 470         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 471             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 472             OID_AUTO, "tx_itr", CTLFLAG_RW,
 473             &ixl_tx_itr, IXL_ITR_4K, "TX ITR");
 474
 475         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 476             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 477             OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
 478             &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
 479
 480 #ifdef IXL_DEBUG_SYSCTL
 481         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 482             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 483             OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
 484             pf, 0, ixl_sysctl_link_status, "A", "Current Link Status");
 485
 486         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 487             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 488             OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
 489             pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
 490
 491         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 492             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 493             OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
 494             pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
 495
 496         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 497             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 498             OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
 499             pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
 500
 501         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 502             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 503             OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
 504             pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
 505 #endif
 506
 507         /* Save off the PCI information */
 508         hw->vendor_id = pci_get_vendor(dev);
 509         hw->device_id = pci_get_device(dev);
 510         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
 511         hw->subsystem_vendor_id =
 512             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 513         hw->subsystem_device_id =
 514             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 515
 516         hw->bus.device = pci_get_slot(dev);
 517         hw->bus.func = pci_get_function(dev);
 518
 519         pf->vc_debug_lvl = 1;
 520
 521         /* Do PCI setup - map BAR0, etc */
 522         if (ixl_allocate_pci_resources(pf)) {
 523                 device_printf(dev, "Allocation of PCI resources failed\n");
 524                 error = ENXIO;
 525                 goto err_out;
 526         }
 527
 528         /* Create for initial debugging use */
 529         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 530             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 531             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
 532             ixl_debug_info, "I", "Debug Information");
 533
 534
 535         /* Establish a clean starting point */
 536         i40e_clear_hw(hw);
 537         error = i40e_pf_reset(hw);
 538         if (error) {
 539                 device_printf(dev,"PF reset failure %x\n", error);
 540                 error = EIO;
 541                 goto err_out;
 542         }
 543
 544         /* Set admin queue parameters */
 545         hw->aq.num_arq_entries = IXL_AQ_LEN;
 546         hw->aq.num_asq_entries = IXL_AQ_LEN;
 547         hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
 548         hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
 549
 550         /* Initialize the shared code */
 551         error = i40e_init_shared_code(hw);
 552         if (error) {
 553                 device_printf(dev,"Unable to initialize the shared code\n");
 554                 error = EIO;
 555                 goto err_out;
 556         }
 557
 558         /* Set up the admin queue */
 559         error = i40e_init_adminq(hw);
 560         if (error) {
 561                 device_printf(dev, "The driver for the device stopped "
 562                     "because the NVM image is newer than expected.\n"
 563                     "You must install the most recent version of "
 564                     " the network driver.\n");
 565                 goto err_out;
 566         }
 567         device_printf(dev, "%s\n", ixl_fw_version_str(hw));
 568
 569         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
 570             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
 571                 device_printf(dev, "The driver for the device detected "
 572                     "a newer version of the NVM image than expected.\n"
 573                     "Please install the most recent version of the network driver.\n");
 574         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
 575             hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
 576                 device_printf(dev, "The driver for the device detected "
 577                     "an older version of the NVM image than expected.\n"
 578                     "Please update the NVM image.\n");
 579
 580         /* Clear PXE mode */
 581         i40e_clear_pxe_mode(hw);
 582
 583         /* Get capabilities from the device */
 584         error = ixl_get_hw_capabilities(pf);
 585         if (error) {
 586                 device_printf(dev, "HW capabilities failure!\n");
 587                 goto err_get_cap;
 588         }
 589
 590         /* Set up host memory cache */
 591         error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 592             hw->func_caps.num_rx_qp, 0, 0);
 593         if (error) {
 594                 device_printf(dev, "init_lan_hmc failed: %d\n", error);
 595                 goto err_get_cap;
 596         }
 597
 598         error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 599         if (error) {
 600                 device_printf(dev, "configure_lan_hmc failed: %d\n", error);
 601                 goto err_mac_hmc;
 602         }
 603
 604         /* Disable LLDP from the firmware */
 605         i40e_aq_stop_lldp(hw, TRUE, NULL);
 606
 607         i40e_get_mac_addr(hw, hw->mac.addr);
 608         error = i40e_validate_mac_addr(hw->mac.addr);
 609         if (error) {
 610                 device_printf(dev, "validate_mac_addr failed: %d\n", error);
 611                 goto err_mac_hmc;
 612         }
 613         bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
 614         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 615
 616         /* Set up VSI and queues */
 617         if (ixl_setup_stations(pf) != 0) {
 618                 device_printf(dev, "setup stations failed!\n");
 619                 error = ENOMEM;
 620                 goto err_mac_hmc;
 621         }
 622
 623         /* Initialize mac filter list for VSI */
 624         SLIST_INIT(&vsi->ftl);
 625
 626         /* Set up interrupt routing here */
 627         if (pf->msix > 1)
 628                 error = ixl_assign_vsi_msix(pf);
 629         else
 630                 error = ixl_assign_vsi_legacy(pf);
 631         if (error)
 632                 goto err_late;
 633
 634         if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 635             (hw->aq.fw_maj_ver < 4)) {
 636                 i40e_msec_delay(75);
 637                 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 638                 if (error)
 639                         device_printf(dev, "link restart failed, aq_err=%d\n",
 640                             pf->hw.aq.asq_last_status);
 641         }
 642
 643         /* Determine link state */
 644         i40e_aq_get_link_info(hw, TRUE, NULL, NULL);
 645         pf->link_up = i40e_get_link_status(hw);
 646
 647         /* Setup OS specific network interface */
 648         if (ixl_setup_interface(dev, vsi) != 0) {
 649                 device_printf(dev, "interface setup failed!\n");
 650                 error = EIO;
 651                 goto err_late;
 652         }
 653
 654         error = ixl_switch_config(pf);
 655         if (error) {
 656                 device_printf(dev, "Initial switch config failed: %d\n", error);
 657                 goto err_mac_hmc;
 658         }
 659
 660         /* Limit phy interrupts to link and modules failure */
 661         error = i40e_aq_set_phy_int_mask(hw,
 662             I40E_AQ_EVENT_LINK_UPDOWN | I40E_AQ_EVENT_MODULE_QUAL_FAIL, NULL);
 663         if (error)
 664                 device_printf(dev, "set phy mask failed: %d\n", error);
 665
 666         /* Get the bus configuration and set the shared code */
 667         bus = ixl_get_bus_info(hw, dev);
 668         i40e_set_pci_config_data(hw, bus);
 669
 670         /* Initialize statistics */
 671         ixl_pf_reset_stats(pf);
 672         ixl_update_stats_counters(pf);
 673         ixl_add_hw_stats(pf);
 674
 675         /* Register for VLAN events */
 676         vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 677             ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 678         vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 679             ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
 680
 681 #ifdef PCI_IOV
 682         /* SR-IOV is only supported when MSI-X is in use. */
 683         if (pf->msix > 1) {
 684                 pf_schema = pci_iov_schema_alloc_node();
 685                 vf_schema = pci_iov_schema_alloc_node();
 686                 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
 687                 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
 688                     IOV_SCHEMA_HASDEFAULT, TRUE);
 689                 pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
 690                     IOV_SCHEMA_HASDEFAULT, FALSE);
 691                 pci_iov_schema_add_bool(vf_schema, "allow-promisc",
 692                     IOV_SCHEMA_HASDEFAULT, FALSE);
 693
 694                 iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
 695                 if (iov_error != 0)
 696                         device_printf(dev,
 697                             "Failed to initialize SR-IOV (error=%d)\n",
 698                             iov_error);
 699         }
 700 #endif
 701
 702 #ifdef DEV_NETMAP
 703         ixl_netmap_attach(vsi);
 704 #endif /* DEV_NETMAP */
 705
 706         INIT_DEBUGOUT("ixl_attach: end");
 707         return (0);
 708
 709 err_late:
 710         if (vsi->ifp != NULL)
 711                 if_free(vsi->ifp);
 712 err_mac_hmc:
 713         i40e_shutdown_lan_hmc(hw);
 714 err_get_cap:
 715         i40e_shutdown_adminq(hw);
 716 err_out:
 717         ixl_free_pci_resources(pf);
 718         ixl_free_vsi(vsi);
 719         IXL_PF_LOCK_DESTROY(pf);
 720         return (error);
 721 }
 722
 723 /*********************************************************************
 724  *  Device removal routine
 725  *
 726  *  The detach entry point is called when the driver is being removed.
 727  *  This routine stops the adapter and deallocates all the resources
 728  *  that were allocated for driver operation.
 729  *
 730  *  return 0 on success, positive on failure
 731  *********************************************************************/
 732
 733 static int
 734 ixl_detach(device_t dev)
 735 {
 736         struct ixl_pf           *pf = device_get_softc(dev);
 737         struct i40e_hw          *hw = &pf->hw;
 738         struct ixl_vsi          *vsi = &pf->vsi;
 739         struct ixl_queue        *que = vsi->queues;
 740         i40e_status             status;
 741 #ifdef PCI_IOV
 742         int                     error;
 743 #endif
 744
 745         INIT_DEBUGOUT("ixl_detach: begin");
 746
 747         /* Make sure VLANS are not using driver */
 748         if (vsi->ifp->if_vlantrunk != NULL) {
 749                 device_printf(dev,"Vlan in use, detach first\n");
 750                 return (EBUSY);
 751         }
 752
 753 #ifdef PCI_IOV
 754         error = pci_iov_detach(dev);
 755         if (error != 0) {
 756                 device_printf(dev, "SR-IOV in use; detach first.\n");
 757                 return (error);
 758         }
 759 #endif
 760
 761         ether_ifdetach(vsi->ifp);
 762         if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) {
 763                 IXL_PF_LOCK(pf);
 764                 ixl_stop(pf);
 765                 IXL_PF_UNLOCK(pf);
 766         }
 767
 768         for (int i = 0; i < vsi->num_queues; i++, que++) {
 769                 if (que->tq) {
 770                         taskqueue_drain(que->tq, &que->task);
 771                         taskqueue_drain(que->tq, &que->tx_task);
 772                         taskqueue_free(que->tq);
 773                 }
 774         }
 775
 776         /* Shutdown LAN HMC */
 777         status = i40e_shutdown_lan_hmc(hw);
 778         if (status)
 779                 device_printf(dev,
 780                     "Shutdown LAN HMC failed with code %d\n", status);
 781
 782         /* Shutdown admin queue */
 783         status = i40e_shutdown_adminq(hw);
 784         if (status)
 785                 device_printf(dev,
 786                     "Shutdown Admin queue failed with code %d\n", status);
 787
 788         /* Unregister VLAN events */
 789         if (vsi->vlan_attach != NULL)
 790                 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
 791         if (vsi->vlan_detach != NULL)
 792                 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
 793
 794         callout_drain(&pf->timer);
 795 #ifdef DEV_NETMAP
 796         netmap_detach(vsi->ifp);
 797 #endif /* DEV_NETMAP */
 798         ixl_free_pci_resources(pf);
 799         bus_generic_detach(dev);
 800         if_free(vsi->ifp);
 801         ixl_free_vsi(vsi);
 802         IXL_PF_LOCK_DESTROY(pf);
 803         return (0);
 804 }
 805
 806 /*********************************************************************
 807  *
 808  *  Shutdown entry point
 809  *
 810  **********************************************************************/
 811
 812 static int
 813 ixl_shutdown(device_t dev)
 814 {
 815         struct ixl_pf *pf = device_get_softc(dev);
 816         IXL_PF_LOCK(pf);
 817         ixl_stop(pf);
 818         IXL_PF_UNLOCK(pf);
 819         return (0);
 820 }
 821
 822
 823 /*********************************************************************
 824  *
 825  *  Get the hardware capabilities
 826  *
 827  **********************************************************************/
 828
 829 static int
 830 ixl_get_hw_capabilities(struct ixl_pf *pf)
 831 {
 832         struct i40e_aqc_list_capabilities_element_resp *buf;
 833         struct i40e_hw  *hw = &pf->hw;
 834         device_t        dev = pf->dev;
 835         int             error, len;
 836         u16             needed;
 837         bool            again = TRUE;
 838
 839         len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
 840 retry:
 841         if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
 842             malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 843                 device_printf(dev, "Unable to allocate cap memory\n");
 844                 return (ENOMEM);
 845         }
 846
 847         /* This populates the hw struct */
 848         error = i40e_aq_discover_capabilities(hw, buf, len,
 849             &needed, i40e_aqc_opc_list_func_capabilities, NULL);
 850         free(buf, M_DEVBUF);
 851         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
 852             (again == TRUE)) {
 853                 /* retry once with a larger buffer */
 854                 again = FALSE;
 855                 len = needed;
 856                 goto retry;
 857         } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
 858                 device_printf(dev, "capability discovery failed: %d\n",
 859                     pf->hw.aq.asq_last_status);
 860                 return (ENODEV);
 861         }
 862
 863         /* Capture this PF's starting queue pair */
 864         pf->qbase = hw->func_caps.base_queue;
 865
 866 #ifdef IXL_DEBUG
 867         device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, "
 868             "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
 869             hw->pf_id, hw->func_caps.num_vfs,
 870             hw->func_caps.num_msix_vectors,
 871             hw->func_caps.num_msix_vectors_vf,
 872             hw->func_caps.fd_filters_guaranteed,
 873             hw->func_caps.fd_filters_best_effort,
 874             hw->func_caps.num_tx_qp,
 875             hw->func_caps.num_rx_qp,
 876             hw->func_caps.base_queue);
 877 #endif
 878         return (error);
 879 }
 880
 881 static void
 882 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
 883 {
 884         device_t        dev = vsi->dev;
 885
 886         /* Enable/disable TXCSUM/TSO4 */
 887         if (!(ifp->if_capenable & IFCAP_TXCSUM)
 888             && !(ifp->if_capenable & IFCAP_TSO4)) {
 889                 if (mask & IFCAP_TXCSUM) {
 890                         ifp->if_capenable |= IFCAP_TXCSUM;
 891                         /* enable TXCSUM, restore TSO if previously enabled */
 892                         if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
 893                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 894                                 ifp->if_capenable |= IFCAP_TSO4;
 895                         }
 896                 }
 897                 else if (mask & IFCAP_TSO4) {
 898                         ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
 899                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
 900                         device_printf(dev,
 901                             "TSO4 requires txcsum, enabling both...\n");
 902                 }
 903         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 904             && !(ifp->if_capenable & IFCAP_TSO4)) {
 905                 if (mask & IFCAP_TXCSUM)
 906                         ifp->if_capenable &= ~IFCAP_TXCSUM;
 907                 else if (mask & IFCAP_TSO4)
 908                         ifp->if_capenable |= IFCAP_TSO4;
 909         } else if((ifp->if_capenable & IFCAP_TXCSUM)
 910             && (ifp->if_capenable & IFCAP_TSO4)) {
 911                 if (mask & IFCAP_TXCSUM) {
 912                         vsi->flags |= IXL_FLAGS_KEEP_TSO4;
 913                         ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
 914                         device_printf(dev,
 915                             "TSO4 requires txcsum, disabling both...\n");
 916                 } else if (mask & IFCAP_TSO4)
 917                         ifp->if_capenable &= ~IFCAP_TSO4;
 918         }
 919
 920         /* Enable/disable TXCSUM_IPV6/TSO6 */
 921         if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 922             && !(ifp->if_capenable & IFCAP_TSO6)) {
 923                 if (mask & IFCAP_TXCSUM_IPV6) {
 924                         ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
 925                         if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
 926                                 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 927                                 ifp->if_capenable |= IFCAP_TSO6;
 928                         }
 929                 } else if (mask & IFCAP_TSO6) {
 930                         ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 931                         vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
 932                         device_printf(dev,
 933                             "TSO6 requires txcsum6, enabling both...\n");
 934                 }
 935         } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 936             && !(ifp->if_capenable & IFCAP_TSO6)) {
 937                 if (mask & IFCAP_TXCSUM_IPV6)
 938                         ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
 939                 else if (mask & IFCAP_TSO6)
 940                         ifp->if_capenable |= IFCAP_TSO6;
 941         } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 942             && (ifp->if_capenable & IFCAP_TSO6)) {
 943                 if (mask & IFCAP_TXCSUM_IPV6) {
 944                         vsi->flags |= IXL_FLAGS_KEEP_TSO6;
 945                         ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
 946                         device_printf(dev,
 947                             "TSO6 requires txcsum6, disabling both...\n");
 948                 } else if (mask & IFCAP_TSO6)
 949                         ifp->if_capenable &= ~IFCAP_TSO6;
 950         }
 951 }
 952
 953 /*********************************************************************
 954  *  Ioctl entry point
 955  *
 956  *  ixl_ioctl is called when the user wants to configure the
 957  *  interface.
 958  *
 959  *  return 0 on success, positive on failure
 960  **********************************************************************/
 961
 962 static int
 963 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 964 {
 965         struct ixl_vsi  *vsi = ifp->if_softc;
 966         struct ixl_pf   *pf = vsi->back;
 967         struct ifreq    *ifr = (struct ifreq *) data;
 968 #if defined(INET) || defined(INET6)
 969         struct ifaddr *ifa = (struct ifaddr *)data;
 970         bool            avoid_reset = FALSE;
 971 #endif
 972         int             error = 0;
 973
 974         switch (command) {
 975
 976         case SIOCSIFADDR:
 977 #ifdef INET
 978                 if (ifa->ifa_addr->sa_family == AF_INET)
 979                         avoid_reset = TRUE;
 980 #endif
 981 #ifdef INET6
 982                 if (ifa->ifa_addr->sa_family == AF_INET6)
 983                         avoid_reset = TRUE;
 984 #endif
 985 #if defined(INET) || defined(INET6)
 986                 /*
 987                 ** Calling init results in link renegotiation,
 988                 ** so we avoid doing it when possible.
 989                 */
 990                 if (avoid_reset) {
 991                         ifp->if_flags |= IFF_UP;
 992                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 993                                 ixl_init(pf);
 994 #ifdef INET
 995                         if (!(ifp->if_flags & IFF_NOARP))
 996                                 arp_ifinit(ifp, ifa);
 997 #endif
 998                 } else
 999                         error = ether_ioctl(ifp, command, data);
1000                 break;
1001 #endif
1002         case SIOCSIFMTU:
1003                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1004                 if (ifr->ifr_mtu > IXL_MAX_FRAME -
1005                    ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
1006                         error = EINVAL;
1007                 } else {
1008                         IXL_PF_LOCK(pf);
1009                         ifp->if_mtu = ifr->ifr_mtu;
1010                         vsi->max_frame_size =
1011                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
1012                             + ETHER_VLAN_ENCAP_LEN;
1013                         ixl_init_locked(pf);
1014                         IXL_PF_UNLOCK(pf);
1015                 }
1016                 break;
1017         case SIOCSIFFLAGS:
1018                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1019                 IXL_PF_LOCK(pf);
1020                 if (ifp->if_flags & IFF_UP) {
1021                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1022                                 if ((ifp->if_flags ^ pf->if_flags) &
1023                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1024                                         ixl_set_promisc(vsi);
1025                                 }
1026                         } else
1027                                 ixl_init_locked(pf);
1028                 } else
1029                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1030                                 ixl_stop(pf);
1031                 pf->if_flags = ifp->if_flags;
1032                 IXL_PF_UNLOCK(pf);
1033                 break;
1034         case SIOCADDMULTI:
1035                 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1036                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1037                         IXL_PF_LOCK(pf);
1038                         ixl_disable_intr(vsi);
1039                         ixl_add_multi(vsi);
1040                         ixl_enable_intr(vsi);
1041                         IXL_PF_UNLOCK(pf);
1042                 }
1043                 break;
1044         case SIOCDELMULTI:
1045                 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1046                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1047                         IXL_PF_LOCK(pf);
1048                         ixl_disable_intr(vsi);
1049                         ixl_del_multi(vsi);
1050                         ixl_enable_intr(vsi);
1051                         IXL_PF_UNLOCK(pf);
1052                 }
1053                 break;
1054         case SIOCSIFMEDIA:
1055         case SIOCGIFMEDIA:
1056                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1057                 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1058                 break;
1059         case SIOCSIFCAP:
1060         {
1061                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1062                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1063
1064                 ixl_cap_txcsum_tso(vsi, ifp, mask);
1065
1066                 if (mask & IFCAP_RXCSUM)
1067                         ifp->if_capenable ^= IFCAP_RXCSUM;
1068                 if (mask & IFCAP_RXCSUM_IPV6)
1069                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1070                 if (mask & IFCAP_LRO)
1071                         ifp->if_capenable ^= IFCAP_LRO;
1072                 if (mask & IFCAP_VLAN_HWTAGGING)
1073                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1074                 if (mask & IFCAP_VLAN_HWFILTER)
1075                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1076                 if (mask & IFCAP_VLAN_HWTSO)
1077                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1078                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1079                         IXL_PF_LOCK(pf);
1080                         ixl_init_locked(pf);
1081                         IXL_PF_UNLOCK(pf);
1082                 }
1083                 VLAN_CAPABILITIES(ifp);
1084
1085                 break;
1086         }
1087
1088         default:
1089                 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1090                 error = ether_ioctl(ifp, command, data);
1091                 break;
1092         }
1093
1094         return (error);
1095 }
1096
1097
1098 /*********************************************************************
1099  *  Init entry point
1100  *
1101  *  This routine is used in two ways. It is used by the stack as
1102  *  init entry point in network interface structure. It is also used
1103  *  by the driver as a hw/sw initialization routine to get to a
1104  *  consistent state.
1105  *
1106  *  return 0 on success, positive on failure
1107  **********************************************************************/
1108
1109 static void
1110 ixl_init_locked(struct ixl_pf *pf)
1111 {
1112         struct i40e_hw  *hw = &pf->hw;
1113         struct ixl_vsi  *vsi = &pf->vsi;
1114         struct ifnet    *ifp = vsi->ifp;
1115         device_t        dev = pf->dev;
1116         struct i40e_filter_control_settings     filter;
1117         u8              tmpaddr[ETHER_ADDR_LEN];
1118         int             ret;
1119
1120         mtx_assert(&pf->pf_mtx, MA_OWNED);
1121         INIT_DEBUGOUT("ixl_init: begin");
1122         ixl_stop(pf);
1123
1124         /* Get the latest mac address... User might use a LAA */
1125         bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1126               I40E_ETH_LENGTH_OF_ADDRESS);
1127         if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1128             i40e_validate_mac_addr(tmpaddr)) {
1129                 bcopy(tmpaddr, hw->mac.addr,
1130                     I40E_ETH_LENGTH_OF_ADDRESS);
1131                 ret = i40e_aq_mac_address_write(hw,
1132                     I40E_AQC_WRITE_TYPE_LAA_ONLY,
1133                     hw->mac.addr, NULL);
1134                 if (ret) {
1135                         device_printf(dev, "LLA address"
1136                          "change failed!!\n");
1137                         return;
1138                 }
1139         }
1140
1141         /* Set the various hardware offload abilities */
1142         ifp->if_hwassist = 0;
1143         if (ifp->if_capenable & IFCAP_TSO)
1144                 ifp->if_hwassist |= CSUM_TSO;
1145         if (ifp->if_capenable & IFCAP_TXCSUM)
1146                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1147         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1148                 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1149
1150         /* Set up the device filtering */
1151         bzero(&filter, sizeof(filter));
1152         filter.enable_ethtype = TRUE;
1153         filter.enable_macvlan = TRUE;
1154 #ifdef IXL_FDIR
1155         filter.enable_fdir = TRUE;
1156 #endif
1157         if (i40e_set_filter_control(hw, &filter))
1158                 device_printf(dev, "set_filter_control() failed\n");
1159
1160         /* Set up RSS */
1161         ixl_config_rss(vsi);
1162
1163         /*
1164         ** Prepare the VSI: rings, hmc contexts, etc...
1165         */
1166         if (ixl_initialize_vsi(vsi)) {
1167                 device_printf(dev, "initialize vsi failed!!\n");
1168                 return;
1169         }
1170
1171         /* Add protocol filters to list */
1172         ixl_init_filters(vsi);
1173
1174         /* Setup vlan's if needed */
1175         ixl_setup_vlan_filters(vsi);
1176
1177         /* Start the local timer */
1178         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1179
1180         /* Set up MSI/X routing and the ITR settings */
1181         if (ixl_enable_msix) {
1182                 ixl_configure_msix(pf);
1183                 ixl_configure_itr(pf);
1184         } else
1185                 ixl_configure_legacy(pf);
1186
1187         ixl_enable_rings(vsi);
1188
1189         i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1190
1191         ixl_reconfigure_filters(vsi);
1192
1193         /* Set MTU in hardware*/
1194         int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size,
1195             TRUE, 0, NULL);
1196         if (aq_error)
1197                 device_printf(vsi->dev,
1198                         "aq_set_mac_config in init error, code %d\n",
1199                     aq_error);
1200
1201         /* And now turn on interrupts */
1202         ixl_enable_intr(vsi);
1203
1204         /* Now inform the stack we're ready */
1205         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1206         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1207
1208         return;
1209 }
1210
1211 static void
1212 ixl_init(void *arg)
1213 {
1214         struct ixl_pf *pf = arg;
1215
1216         IXL_PF_LOCK(pf);
1217         ixl_init_locked(pf);
1218         IXL_PF_UNLOCK(pf);
1219         return;
1220 }
1221
1222 /*
1223 **
1224 ** MSIX Interrupt Handlers and Tasklets
1225 **
1226 */
1227 static void
1228 ixl_handle_que(void *context, int pending)
1229 {
1230         struct ixl_queue *que = context;
1231         struct ixl_vsi *vsi = que->vsi;
1232         struct i40e_hw  *hw = vsi->hw;
1233         struct tx_ring  *txr = &que->txr;
1234         struct ifnet    *ifp = vsi->ifp;
1235         bool            more;
1236
1237         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1238                 more = ixl_rxeof(que, IXL_RX_LIMIT);
1239                 IXL_TX_LOCK(txr);
1240                 ixl_txeof(que);
1241                 if (!drbr_empty(ifp, txr->br))
1242                         ixl_mq_start_locked(ifp, txr);
1243                 IXL_TX_UNLOCK(txr);
1244                 if (more) {
1245                         taskqueue_enqueue(que->tq, &que->task);
1246                         return;
1247                 }
1248         }
1249
1250         /* Reenable this interrupt - hmmm */
1251         ixl_enable_queue(hw, que->me);
1252         return;
1253 }
1254
1255
1256 /*********************************************************************
1257  *
1258  *  Legacy Interrupt Service routine
1259  *
1260  **********************************************************************/
1261 void
1262 ixl_intr(void *arg)
1263 {
1264         struct ixl_pf           *pf = arg;
1265         struct i40e_hw          *hw =  &pf->hw;
1266         struct ixl_vsi          *vsi = &pf->vsi;
1267         struct ixl_queue        *que = vsi->queues;
1268         struct ifnet            *ifp = vsi->ifp;
1269         struct tx_ring          *txr = &que->txr;
1270         u32                     reg, icr0, mask;
1271         bool                    more_tx, more_rx;
1272
1273         ++que->irqs;
1274
1275         /* Protect against spurious interrupts */
1276         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1277                 return;
1278
1279         icr0 = rd32(hw, I40E_PFINT_ICR0);
1280
1281         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1282         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1283         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1284
1285         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1286
1287 #ifdef PCI_IOV
1288         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1289                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1290 #endif
1291
1292         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1293                 taskqueue_enqueue(pf->tq, &pf->adminq);
1294                 return;
1295         }
1296
1297         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1298
1299         IXL_TX_LOCK(txr);
1300         more_tx = ixl_txeof(que);
1301         if (!drbr_empty(vsi->ifp, txr->br))
1302                 more_tx = 1;
1303         IXL_TX_UNLOCK(txr);
1304
1305         /* re-enable other interrupt causes */
1306         wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1307
1308         /* And now the queues */
1309         reg = rd32(hw, I40E_QINT_RQCTL(0));
1310         reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1311         wr32(hw, I40E_QINT_RQCTL(0), reg);
1312
1313         reg = rd32(hw, I40E_QINT_TQCTL(0));
1314         reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1315         reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1316         wr32(hw, I40E_QINT_TQCTL(0), reg);
1317
1318         ixl_enable_legacy(hw);
1319
1320         return;
1321 }
1322
1323
1324 /*********************************************************************
1325  *
1326  *  MSIX VSI Interrupt Service routine
1327  *
1328  **********************************************************************/
1329 void
1330 ixl_msix_que(void *arg)
1331 {
1332         struct ixl_queue        *que = arg;
1333         struct ixl_vsi  *vsi = que->vsi;
1334         struct i40e_hw  *hw = vsi->hw;
1335         struct tx_ring  *txr = &que->txr;
1336         bool            more_tx, more_rx;
1337
1338         /* Protect against spurious interrupts */
1339         if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1340                 return;
1341
1342         ++que->irqs;
1343
1344         more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1345
1346         IXL_TX_LOCK(txr);
1347         more_tx = ixl_txeof(que);
1348         /*
1349         ** Make certain that if the stack
1350         ** has anything queued the task gets
1351         ** scheduled to handle it.
1352         */
1353         if (!drbr_empty(vsi->ifp, txr->br))
1354                 more_tx = 1;
1355         IXL_TX_UNLOCK(txr);
1356
1357         ixl_set_queue_rx_itr(que);
1358         ixl_set_queue_tx_itr(que);
1359
1360         if (more_tx || more_rx)
1361                 taskqueue_enqueue(que->tq, &que->task);
1362         else
1363                 ixl_enable_queue(hw, que->me);
1364
1365         return;
1366 }
1367
1368
1369 /*********************************************************************
1370  *
1371  *  MSIX Admin Queue Interrupt Service routine
1372  *
1373  **********************************************************************/
1374 static void
1375 ixl_msix_adminq(void *arg)
1376 {
1377         struct ixl_pf   *pf = arg;
1378         struct i40e_hw  *hw = &pf->hw;
1379         u32             reg, mask;
1380
1381         ++pf->admin_irq;
1382
1383         reg = rd32(hw, I40E_PFINT_ICR0);
1384         mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1385
1386         /* Check on the cause */
1387         if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
1388                 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
1389
1390         if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1391                 ixl_handle_mdd_event(pf);
1392                 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
1393         }
1394
1395 #ifdef PCI_IOV
1396         if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1397                 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1398                 taskqueue_enqueue(pf->tq, &pf->vflr_task);
1399         }
1400 #endif
1401
1402         reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1403         reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1404         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1405
1406         taskqueue_enqueue(pf->tq, &pf->adminq);
1407         return;
1408 }
1409
1410 /*********************************************************************
1411  *
1412  *  Media Ioctl callback
1413  *
1414  *  This routine is called whenever the user queries the status of
1415  *  the interface using ifconfig.
1416  *
1417  **********************************************************************/
1418 static void
1419 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1420 {
1421         struct ixl_vsi  *vsi = ifp->if_softc;
1422         struct ixl_pf   *pf = vsi->back;
1423         struct i40e_hw  *hw = &pf->hw;
1424
1425         INIT_DEBUGOUT("ixl_media_status: begin");
1426         IXL_PF_LOCK(pf);
1427
1428         hw->phy.get_link_info = TRUE;
1429         pf->link_up = i40e_get_link_status(hw);
1430         ixl_update_link_status(pf);
1431
1432         ifmr->ifm_status = IFM_AVALID;
1433         ifmr->ifm_active = IFM_ETHER;
1434
1435         if (!pf->link_up) {
1436                 IXL_PF_UNLOCK(pf);
1437                 return;
1438         }
1439
1440         ifmr->ifm_status |= IFM_ACTIVE;
1441         /* Hardware is always full-duplex */
1442         ifmr->ifm_active |= IFM_FDX;
1443
1444         switch (hw->phy.link_info.phy_type) {
1445                 /* 100 M */
1446                 case I40E_PHY_TYPE_100BASE_TX:
1447                         ifmr->ifm_active |= IFM_100_TX;
1448                         break;
1449                 /* 1 G */
1450                 case I40E_PHY_TYPE_1000BASE_T:
1451                         ifmr->ifm_active |= IFM_1000_T;
1452                         break;
1453                 case I40E_PHY_TYPE_1000BASE_SX:
1454                         ifmr->ifm_active |= IFM_1000_SX;
1455                         break;
1456                 case I40E_PHY_TYPE_1000BASE_LX:
1457                         ifmr->ifm_active |= IFM_1000_LX;
1458                         break;
1459                 /* 10 G */
1460                 case I40E_PHY_TYPE_10GBASE_CR1:
1461                 case I40E_PHY_TYPE_10GBASE_CR1_CU:
1462                 case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1463                 /* Using this until a real KR media type */
1464                 case I40E_PHY_TYPE_10GBASE_KR:
1465                 case I40E_PHY_TYPE_10GBASE_KX4:
1466                         ifmr->ifm_active |= IFM_10G_TWINAX;
1467                         break;
1468                 case I40E_PHY_TYPE_10GBASE_SR:
1469                         ifmr->ifm_active |= IFM_10G_SR;
1470                         break;
1471                 case I40E_PHY_TYPE_10GBASE_LR:
1472                         ifmr->ifm_active |= IFM_10G_LR;
1473                         break;
1474                 case I40E_PHY_TYPE_10GBASE_T:
1475                         ifmr->ifm_active |= IFM_10G_T;
1476                         break;
1477                 /* 40 G */
1478                 case I40E_PHY_TYPE_40GBASE_CR4:
1479                 case I40E_PHY_TYPE_40GBASE_CR4_CU:
1480                         ifmr->ifm_active |= IFM_40G_CR4;
1481                         break;
1482                 case I40E_PHY_TYPE_40GBASE_SR4:
1483                         ifmr->ifm_active |= IFM_40G_SR4;
1484                         break;
1485                 case I40E_PHY_TYPE_40GBASE_LR4:
1486                         ifmr->ifm_active |= IFM_40G_LR4;
1487                         break;
1488                 /*
1489                 ** Set these to CR4 because OS does not
1490                 ** have types available yet.
1491                 */
1492                 case I40E_PHY_TYPE_40GBASE_KR4:
1493                 case I40E_PHY_TYPE_XLAUI:
1494                 case I40E_PHY_TYPE_XLPPI:
1495                 case I40E_PHY_TYPE_40GBASE_AOC:
1496                         ifmr->ifm_active |= IFM_40G_CR4;
1497                         break;
1498                 default:
1499                         ifmr->ifm_active |= IFM_UNKNOWN;
1500                         break;
1501         }
1502         /* Report flow control status as well */
1503         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1504                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1505         if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1506                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1507
1508         IXL_PF_UNLOCK(pf);
1509
1510         return;
1511 }
1512
1513 /*********************************************************************
1514  *
1515  *  Media Ioctl callback
1516  *
1517  *  This routine is called when the user changes speed/duplex using
1518  *  media/mediopt option with ifconfig.
1519  *
1520  **********************************************************************/
1521 static int
1522 ixl_media_change(struct ifnet * ifp)
1523 {
1524         struct ixl_vsi *vsi = ifp->if_softc;
1525         struct ifmedia *ifm = &vsi->media;
1526
1527         INIT_DEBUGOUT("ixl_media_change: begin");
1528
1529         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1530                 return (EINVAL);
1531
1532         if_printf(ifp, "Media change is currently not supported.\n");
1533
1534         return (ENODEV);
1535 }
1536
1537
1538 #ifdef IXL_FDIR
1539 /*
1540 ** ATR: Application Targetted Receive - creates a filter
1541 **      based on TX flow info that will keep the receive
1542 **      portion of the flow on the same queue. Based on the
1543 **      implementation this is only available for TCP connections
1544 */
1545 void
1546 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1547 {
1548         struct ixl_vsi                  *vsi = que->vsi;
1549         struct tx_ring                  *txr = &que->txr;
1550         struct i40e_filter_program_desc *FDIR;
1551         u32                             ptype, dtype;
1552         int                             idx;
1553
1554         /* check if ATR is enabled and sample rate */
1555         if ((!ixl_enable_fdir) || (!txr->atr_rate))
1556                 return;
1557         /*
1558         ** We sample all TCP SYN/FIN packets,
1559         ** or at the selected sample rate
1560         */
1561         txr->atr_count++;
1562         if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1563             (txr->atr_count < txr->atr_rate))
1564                 return;
1565         txr->atr_count = 0;
1566
1567         /* Get a descriptor to use */
1568         idx = txr->next_avail;
1569         FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1570         if (++idx == que->num_desc)
1571                 idx = 0;
1572         txr->avail--;
1573         txr->next_avail = idx;
1574
1575         ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1576             I40E_TXD_FLTR_QW0_QINDEX_MASK;
1577
1578         ptype |= (etype == ETHERTYPE_IP) ?
1579             (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1580             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1581             (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1582             I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1583
1584         ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1585
1586         dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1587
1588         /*
1589         ** We use the TCP TH_FIN as a trigger to remove
1590         ** the filter, otherwise its an update.
1591         */
1592         dtype |= (th->th_flags & TH_FIN) ?
1593             (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1594             I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1595             (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1596             I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1597
1598         dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1599             I40E_TXD_FLTR_QW1_DEST_SHIFT;
1600
1601         dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1602             I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1603
1604         FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1605         FDIR->dtype_cmd_cntindex = htole32(dtype);
1606         return;
1607 }
1608 #endif
1609
1610
1611 static void
1612 ixl_set_promisc(struct ixl_vsi *vsi)
1613 {
1614         struct ifnet    *ifp = vsi->ifp;
1615         struct i40e_hw  *hw = vsi->hw;
1616         int             err, mcnt = 0;
1617         bool            uni = FALSE, multi = FALSE;
1618
1619         if (ifp->if_flags & IFF_ALLMULTI)
1620                 multi = TRUE;
1621         else { /* Need to count the multicast addresses */
1622                 struct  ifmultiaddr *ifma;
1623                 if_maddr_rlock(ifp);
1624                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1625                         if (ifma->ifma_addr->sa_family != AF_LINK)
1626                                 continue;
1627                         if (mcnt == MAX_MULTICAST_ADDR)
1628                                 break;
1629                         mcnt++;
1630                 }
1631                 if_maddr_runlock(ifp);
1632         }
1633
1634         if (mcnt >= MAX_MULTICAST_ADDR)
1635                 multi = TRUE;
1636         if (ifp->if_flags & IFF_PROMISC)
1637                 uni = TRUE;
1638
1639         err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1640             vsi->seid, uni, NULL);
1641         err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1642             vsi->seid, multi, NULL);
1643         return;
1644 }
1645
1646 /*********************************************************************
1647  *      Filter Routines
1648  *
1649  *      Routines for multicast and vlan filter management.
1650  *
1651  *********************************************************************/
1652 static void
1653 ixl_add_multi(struct ixl_vsi *vsi)
1654 {
1655         struct  ifmultiaddr     *ifma;
1656         struct ifnet            *ifp = vsi->ifp;
1657         struct i40e_hw          *hw = vsi->hw;
1658         int                     mcnt = 0, flags;
1659
1660         IOCTL_DEBUGOUT("ixl_add_multi: begin");
1661
1662         if_maddr_rlock(ifp);
1663         /*
1664         ** First just get a count, to decide if we
1665         ** we simply use multicast promiscuous.
1666         */
1667         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1668                 if (ifma->ifma_addr->sa_family != AF_LINK)
1669                         continue;
1670                 mcnt++;
1671         }
1672         if_maddr_runlock(ifp);
1673
1674         if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1675                 /* delete existing MC filters */
1676                 ixl_del_hw_filters(vsi, mcnt);
1677                 i40e_aq_set_vsi_multicast_promiscuous(hw,
1678                     vsi->seid, TRUE, NULL);
1679                 return;
1680         }
1681
1682         mcnt = 0;
1683         if_maddr_rlock(ifp);
1684         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1685                 if (ifma->ifma_addr->sa_family != AF_LINK)
1686                         continue;
1687                 ixl_add_mc_filter(vsi,
1688                     (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1689                 mcnt++;
1690         }
1691         if_maddr_runlock(ifp);
1692         if (mcnt > 0) {
1693                 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1694                 ixl_add_hw_filters(vsi, flags, mcnt);
1695         }
1696
1697         IOCTL_DEBUGOUT("ixl_add_multi: end");
1698         return;
1699 }
1700
1701 static void
1702 ixl_del_multi(struct ixl_vsi *vsi)
1703 {
1704         struct ifnet            *ifp = vsi->ifp;
1705         struct ifmultiaddr      *ifma;
1706         struct ixl_mac_filter   *f;
1707         int                     mcnt = 0;
1708         bool            match = FALSE;
1709
1710         IOCTL_DEBUGOUT("ixl_del_multi: begin");
1711
1712         /* Search for removed multicast addresses */
1713         if_maddr_rlock(ifp);
1714         SLIST_FOREACH(f, &vsi->ftl, next) {
1715                 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1716                         match = FALSE;
1717                         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1718                                 if (ifma->ifma_addr->sa_family != AF_LINK)
1719                                         continue;
1720                                 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
1721                                 if (cmp_etheraddr(f->macaddr, mc_addr)) {
1722                                         match = TRUE;
1723                                         break;
1724                                 }
1725                         }
1726                         if (match == FALSE) {
1727                                 f->flags |= IXL_FILTER_DEL;
1728                                 mcnt++;
1729                         }
1730                 }
1731         }
1732         if_maddr_runlock(ifp);
1733
1734         if (mcnt > 0)
1735                 ixl_del_hw_filters(vsi, mcnt);
1736 }
1737
1738
1739 /*********************************************************************
1740  *  Timer routine
1741  *
1742  *  This routine checks for link status,updates statistics,
1743  *  and runs the watchdog check.
1744  *
1745  **********************************************************************/
1746
1747 static void
1748 ixl_local_timer(void *arg)
1749 {
1750         struct ixl_pf           *pf = arg;
1751         struct i40e_hw          *hw = &pf->hw;
1752         struct ixl_vsi          *vsi = &pf->vsi;
1753         struct ixl_queue        *que = vsi->queues;
1754         device_t                dev = pf->dev;
1755         int                     hung = 0;
1756         u32                     mask;
1757
1758         mtx_assert(&pf->pf_mtx, MA_OWNED);
1759
1760         /* Fire off the adminq task */
1761         taskqueue_enqueue(pf->tq, &pf->adminq);
1762
1763         /* Update stats */
1764         ixl_update_stats_counters(pf);
1765
1766         /*
1767         ** Check status of the queues
1768         */
1769         mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
1770                 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
1771
1772         for (int i = 0; i < vsi->num_queues; i++,que++) {
1773                 /* Any queues with outstanding work get a sw irq */
1774                 if (que->busy)
1775                         wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
1776                 /*
1777                 ** Each time txeof runs without cleaning, but there
1778                 ** are uncleaned descriptors it increments busy. If
1779                 ** we get to 5 we declare it hung.
1780                 */
1781                 if (que->busy == IXL_QUEUE_HUNG) {
1782                         ++hung;
1783                         /* Mark the queue as inactive */
1784                         vsi->active_queues &= ~((u64)1 << que->me);
1785                         continue;
1786                 } else {
1787                         /* Check if we've come back from hung */
1788                         if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
1789                                 vsi->active_queues |= ((u64)1 << que->me);
1790                 }
1791                 if (que->busy >= IXL_MAX_TX_BUSY) {
1792 #ifdef IXL_DEBUG
1793                         device_printf(dev,"Warning queue %d "
1794                             "appears to be hung!\n", i);
1795 #endif
1796                         que->busy = IXL_QUEUE_HUNG;
1797                         ++hung;
1798                 }
1799         }
1800         /* Only reinit if all queues show hung */
1801         if (hung == vsi->num_queues)
1802                 goto hung;
1803
1804         callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1805         return;
1806
1807 hung:
1808         device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
1809         ixl_init_locked(pf);
1810 }
1811
1812 /*
1813 ** Note: this routine updates the OS on the link state
1814 **      the real check of the hardware only happens with
1815 **      a link interrupt.
1816 */
1817 static void
1818 ixl_update_link_status(struct ixl_pf *pf)
1819 {
1820         struct ixl_vsi          *vsi = &pf->vsi;
1821         struct i40e_hw          *hw = &pf->hw;
1822         struct ifnet            *ifp = vsi->ifp;
1823         device_t                dev = pf->dev;
1824
1825         if (pf->link_up){
1826                 if (vsi->link_active == FALSE) {
1827                         pf->fc = hw->fc.current_mode;
1828                         if (bootverbose) {
1829                                 device_printf(dev,"Link is up %d Gbps %s,"
1830                                     " Flow Control: %s\n",
1831                                     ((pf->link_speed ==
1832                                     I40E_LINK_SPEED_40GB)? 40:10),
1833                                     "Full Duplex", ixl_fc_string[pf->fc]);
1834                         }
1835                         vsi->link_active = TRUE;
1836                         /*
1837                         ** Warn user if link speed on NPAR enabled
1838                         ** partition is not at least 10GB
1839                         */
1840                         if (hw->func_caps.npar_enable &&
1841                            (hw->phy.link_info.link_speed ==
1842                            I40E_LINK_SPEED_1GB ||
1843                            hw->phy.link_info.link_speed ==
1844                            I40E_LINK_SPEED_100MB))
1845                                 device_printf(dev, "The partition detected"
1846                                     "link speed that is less than 10Gbps\n");
1847                         if_link_state_change(ifp, LINK_STATE_UP);
1848                 }
1849         } else { /* Link down */
1850                 if (vsi->link_active == TRUE) {
1851                         if (bootverbose)
1852                                 device_printf(dev,"Link is Down\n");
1853                         if_link_state_change(ifp, LINK_STATE_DOWN);
1854                         vsi->link_active = FALSE;
1855                 }
1856         }
1857
1858         return;
1859 }
1860
1861 /*********************************************************************
1862  *
1863  *  This routine disables all traffic on the adapter by issuing a
1864  *  global reset on the MAC and deallocates TX/RX buffers.
1865  *
1866  **********************************************************************/
1867
1868 static void
1869 ixl_stop(struct ixl_pf *pf)
1870 {
1871         struct ixl_vsi  *vsi = &pf->vsi;
1872         struct ifnet    *ifp = vsi->ifp;
1873
1874         mtx_assert(&pf->pf_mtx, MA_OWNED);
1875
1876         INIT_DEBUGOUT("ixl_stop: begin\n");
1877         if (pf->num_vfs == 0)
1878                 ixl_disable_intr(vsi);
1879         else
1880                 ixl_disable_rings_intr(vsi);
1881         ixl_disable_rings(vsi);
1882
1883         /* Tell the stack that the interface is no longer active */
1884         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1885
1886         /* Stop the local timer */
1887         callout_stop(&pf->timer);
1888
1889         return;
1890 }
1891
1892
1893 /*********************************************************************
1894  *
1895  *  Setup MSIX Interrupt resources and handlers for the VSI
1896  *
1897  **********************************************************************/
1898 static int
1899 ixl_assign_vsi_legacy(struct ixl_pf *pf)
1900 {
1901         device_t        dev = pf->dev;
1902         struct          ixl_vsi *vsi = &pf->vsi;
1903         struct          ixl_queue *que = vsi->queues;
1904         int             error, rid = 0;
1905
1906         if (pf->msix == 1)
1907                 rid = 1;
1908         pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1909             &rid, RF_SHAREABLE | RF_ACTIVE);
1910         if (pf->res == NULL) {
1911                 device_printf(dev,"Unable to allocate"
1912                     " bus resource: vsi legacy/msi interrupt\n");
1913                 return (ENXIO);
1914         }
1915
1916         /* Set the handler function */
1917         error = bus_setup_intr(dev, pf->res,
1918             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1919             ixl_intr, pf, &pf->tag);
1920         if (error) {
1921                 pf->res = NULL;
1922                 device_printf(dev, "Failed to register legacy/msi handler");
1923                 return (error);
1924         }
1925         bus_describe_intr(dev, pf->res, pf->tag, "irq0");
1926         TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
1927         TASK_INIT(&que->task, 0, ixl_handle_que, que);
1928         que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
1929             taskqueue_thread_enqueue, &que->tq);
1930         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1931             device_get_nameunit(dev));
1932         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1933
1934 #ifdef PCI_IOV
1935         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1936 #endif
1937
1938         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1939             taskqueue_thread_enqueue, &pf->tq);
1940         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1941             device_get_nameunit(dev));
1942
1943         return (0);
1944 }
1945
1946
1947 /*********************************************************************
1948  *
1949  *  Setup MSIX Interrupt resources and handlers for the VSI
1950  *
1951  **********************************************************************/
1952 static int
1953 ixl_assign_vsi_msix(struct ixl_pf *pf)
1954 {
1955         device_t        dev = pf->dev;
1956         struct          ixl_vsi *vsi = &pf->vsi;
1957         struct          ixl_queue *que = vsi->queues;
1958         struct          tx_ring  *txr;
1959         int             error, rid, vector = 0;
1960 #ifdef  RSS
1961         cpuset_t cpu_mask;
1962 #endif
1963
1964         /* Admin Que is vector 0*/
1965         rid = vector + 1;
1966         pf->res = bus_alloc_resource_any(dev,
1967             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
1968         if (!pf->res) {
1969                 device_printf(dev,"Unable to allocate"
1970             " bus resource: Adminq interrupt [%d]\n", rid);
1971                 return (ENXIO);
1972         }
1973         /* Set the adminq vector and handler */
1974         error = bus_setup_intr(dev, pf->res,
1975             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1976             ixl_msix_adminq, pf, &pf->tag);
1977         if (error) {
1978                 pf->res = NULL;
1979                 device_printf(dev, "Failed to register Admin que handler");
1980                 return (error);
1981         }
1982         bus_describe_intr(dev, pf->res, pf->tag, "aq");
1983         pf->admvec = vector;
1984         /* Tasklet for Admin Queue */
1985         TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1986
1987 #ifdef PCI_IOV
1988         TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1989 #endif
1990
1991         pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1992             taskqueue_thread_enqueue, &pf->tq);
1993         taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1994             device_get_nameunit(pf->dev));
1995         ++vector;
1996
1997         /* Now set up the stations */
1998         for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
1999                 int cpu_id = i;
2000                 rid = vector + 1;
2001                 txr = &que->txr;
2002                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2003                     RF_SHAREABLE | RF_ACTIVE);
2004                 if (que->res == NULL) {
2005                         device_printf(dev,"Unable to allocate"
2006                             " bus resource: que interrupt [%d]\n", vector);
2007                         return (ENXIO);
2008                 }
2009                 /* Set the handler function */
2010                 error = bus_setup_intr(dev, que->res,
2011                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2012                     ixl_msix_que, que, &que->tag);
2013                 if (error) {
2014                         que->res = NULL;
2015                         device_printf(dev, "Failed to register que handler");
2016                         return (error);
2017                 }
2018                 bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2019                 /* Bind the vector to a CPU */
2020 #ifdef RSS
2021                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2022 #endif
2023                 bus_bind_intr(dev, que->res, cpu_id);
2024                 que->msix = vector;
2025                 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2026                 TASK_INIT(&que->task, 0, ixl_handle_que, que);
2027                 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2028                     taskqueue_thread_enqueue, &que->tq);
2029 #ifdef RSS
2030                 CPU_SETOF(cpu_id, &cpu_mask);
2031                 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2032                     &cpu_mask, "%s (bucket %d)",
2033                     device_get_nameunit(dev), cpu_id);
2034 #else
2035                 taskqueue_start_threads(&que->tq, 1, PI_NET,
2036                     "%s que", device_get_nameunit(dev));
2037 #endif
2038         }
2039
2040         return (0);
2041 }
2042
2043
2044 /*
2045  * Allocate MSI/X vectors
2046  */
2047 static int
2048 ixl_init_msix(struct ixl_pf *pf)
2049 {
2050         device_t dev = pf->dev;
2051         int rid, want, vectors, queues, available;
2052
2053         /* Override by tuneable */
2054         if (ixl_enable_msix == 0)
2055                 goto msi;
2056
2057         /*
2058         ** When used in a virtualized environment
2059         ** PCI BUSMASTER capability may not be set
2060         ** so explicity set it here and rewrite
2061         ** the ENABLE in the MSIX control register
2062         ** at this point to cause the host to
2063         ** successfully initialize us.
2064         */
2065         {
2066                 u16 pci_cmd_word;
2067                 int msix_ctrl;
2068                 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2069                 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2070                 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2071                 pci_find_cap(dev, PCIY_MSIX, &rid);
2072                 rid += PCIR_MSIX_CTRL;
2073                 msix_ctrl = pci_read_config(dev, rid, 2);
2074                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2075                 pci_write_config(dev, rid, msix_ctrl, 2);
2076         }
2077
2078         /* First try MSI/X */
2079         rid = PCIR_BAR(IXL_BAR);
2080         pf->msix_mem = bus_alloc_resource_any(dev,
2081             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2082         if (!pf->msix_mem) {
2083                 /* May not be enabled */
2084                 device_printf(pf->dev,
2085                     "Unable to map MSIX table \n");
2086                 goto msi;
2087         }
2088
2089         available = pci_msix_count(dev);
2090         if (available == 0) { /* system has msix disabled */
2091                 bus_release_resource(dev, SYS_RES_MEMORY,
2092                     rid, pf->msix_mem);
2093                 pf->msix_mem = NULL;
2094                 goto msi;
2095         }
2096
2097         /* Figure out a reasonable auto config value */
2098         queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2099
2100         /* Override with hardcoded value if sane */
2101         if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2102                 queues = ixl_max_queues;
2103
2104 #ifdef  RSS
2105         /* If we're doing RSS, clamp at the number of RSS buckets */
2106         if (queues > rss_getnumbuckets())
2107                 queues = rss_getnumbuckets();
2108 #endif
2109
2110         /*
2111         ** Want one vector (RX/TX pair) per queue
2112         ** plus an additional for the admin queue.
2113         */
2114         want = queues + 1;
2115         if (want <= available)  /* Have enough */
2116                 vectors = want;
2117         else {
2118                 device_printf(pf->dev,
2119                     "MSIX Configuration Problem, "
2120                     "%d vectors available but %d wanted!\n",
2121                     available, want);
2122                 return (0); /* Will go to Legacy setup */
2123         }
2124
2125         if (pci_alloc_msix(dev, &vectors) == 0) {
2126                 device_printf(pf->dev,
2127                     "Using MSIX interrupts with %d vectors\n", vectors);
2128                 pf->msix = vectors;
2129                 pf->vsi.num_queues = queues;
2130 #ifdef RSS
2131                 /*
2132                  * If we're doing RSS, the number of queues needs to
2133                  * match the number of RSS buckets that are configured.
2134                  *
2135                  * + If there's more queues than RSS buckets, we'll end
2136                  *   up with queues that get no traffic.
2137                  *
2138                  * + If there's more RSS buckets than queues, we'll end
2139                  *   up having multiple RSS buckets map to the same queue,
2140                  *   so there'll be some contention.
2141                  */
2142                 if (queues != rss_getnumbuckets()) {
2143                         device_printf(dev,
2144                             "%s: queues (%d) != RSS buckets (%d)"
2145                             "; performance will be impacted.\n",
2146                             __func__, queues, rss_getnumbuckets());
2147                 }
2148 #endif
2149                 return (vectors);
2150         }
2151 msi:
2152         vectors = pci_msi_count(dev);
2153         pf->vsi.num_queues = 1;
2154         pf->msix = 1;
2155         ixl_max_queues = 1;
2156         ixl_enable_msix = 0;
2157         if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2158                 device_printf(pf->dev,"Using an MSI interrupt\n");
2159         else {
2160                 pf->msix = 0;
2161                 device_printf(pf->dev,"Using a Legacy interrupt\n");
2162         }
2163         return (vectors);
2164 }
2165
2166
2167 /*
2168  * Plumb MSI/X vectors
2169  */
2170 static void
2171 ixl_configure_msix(struct ixl_pf *pf)
2172 {
2173         struct i40e_hw  *hw = &pf->hw;
2174         struct ixl_vsi *vsi = &pf->vsi;
2175         u32             reg;
2176         u16             vector = 1;
2177
2178         /* First set up the adminq - vector 0 */
2179         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2180         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2181
2182         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2183             I40E_PFINT_ICR0_ENA_GRST_MASK |
2184             I40E_PFINT_ICR0_HMC_ERR_MASK |
2185             I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2186             I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2187             I40E_PFINT_ICR0_ENA_VFLR_MASK |
2188             I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2189         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2190
2191         wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2192         wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E);
2193
2194         wr32(hw, I40E_PFINT_DYN_CTL0,
2195             I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2196             I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2197
2198         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2199
2200         /* Next configure the queues */
2201         for (int i = 0; i < vsi->num_queues; i++, vector++) {
2202                 wr32(hw, I40E_PFINT_DYN_CTLN(i), i);
2203                 wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2204
2205                 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2206                 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2207                 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2208                 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2209                 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2210                 wr32(hw, I40E_QINT_RQCTL(i), reg);
2211
2212                 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2213                 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2214                 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2215                 ((i+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2216                 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2217                 if (i == (vsi->num_queues - 1))
2218                         reg |= (IXL_QUEUE_EOL
2219                             << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2220                 wr32(hw, I40E_QINT_TQCTL(i), reg);
2221         }
2222 }
2223
2224 /*
2225  * Configure for MSI single vector operation
2226  */
2227 static void
2228 ixl_configure_legacy(struct ixl_pf *pf)
2229 {
2230         struct i40e_hw  *hw = &pf->hw;
2231         u32             reg;
2232
2233
2234         wr32(hw, I40E_PFINT_ITR0(0), 0);
2235         wr32(hw, I40E_PFINT_ITR0(1), 0);
2236
2237
2238         /* Setup "other" causes */
2239         reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2240             | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2241             | I40E_PFINT_ICR0_ENA_GRST_MASK
2242             | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2243             | I40E_PFINT_ICR0_ENA_GPIO_MASK
2244             | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2245             | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2246             | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2247             | I40E_PFINT_ICR0_ENA_VFLR_MASK
2248             | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2249             ;
2250         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2251
2252         /* SW_ITR_IDX = 0, but don't change INTENA */
2253         wr32(hw, I40E_PFINT_DYN_CTL0,
2254             I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2255             I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2256         /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2257         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2258
2259         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2260         wr32(hw, I40E_PFINT_LNKLST0, 0);
2261
2262         /* Associate the queue pair to the vector and enable the q int */
2263         reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2264             | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2265             | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2266         wr32(hw, I40E_QINT_RQCTL(0), reg);
2267
2268         reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2269             | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2270             | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2271         wr32(hw, I40E_QINT_TQCTL(0), reg);
2272
2273         /* Next enable the queue pair */
2274         reg = rd32(hw, I40E_QTX_ENA(0));
2275         reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2276         wr32(hw, I40E_QTX_ENA(0), reg);
2277
2278         reg = rd32(hw, I40E_QRX_ENA(0));
2279         reg |= I40E_QRX_ENA_QENA_REQ_MASK;
2280         wr32(hw, I40E_QRX_ENA(0), reg);
2281 }
2282
2283
2284 /*
2285  * Set the Initial ITR state
2286  */
2287 static void
2288 ixl_configure_itr(struct ixl_pf *pf)
2289 {
2290         struct i40e_hw          *hw = &pf->hw;
2291         struct ixl_vsi          *vsi = &pf->vsi;
2292         struct ixl_queue        *que = vsi->queues;
2293
2294         vsi->rx_itr_setting = ixl_rx_itr;
2295         if (ixl_dynamic_rx_itr)
2296                 vsi->rx_itr_setting |= IXL_ITR_DYNAMIC;
2297         vsi->tx_itr_setting = ixl_tx_itr;
2298         if (ixl_dynamic_tx_itr)
2299                 vsi->tx_itr_setting |= IXL_ITR_DYNAMIC;
2300
2301         for (int i = 0; i < vsi->num_queues; i++, que++) {
2302                 struct tx_ring  *txr = &que->txr;
2303                 struct rx_ring  *rxr = &que->rxr;
2304
2305                 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2306                     vsi->rx_itr_setting);
2307                 rxr->itr = vsi->rx_itr_setting;
2308                 rxr->latency = IXL_AVE_LATENCY;
2309                 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2310                     vsi->tx_itr_setting);
2311                 txr->itr = vsi->tx_itr_setting;
2312                 txr->latency = IXL_AVE_LATENCY;
2313         }
2314 }
2315
2316
2317 static int
2318 ixl_allocate_pci_resources(struct ixl_pf *pf)
2319 {
2320         int             rid;
2321         device_t        dev = pf->dev;
2322
2323         rid = PCIR_BAR(0);
2324         pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2325             &rid, RF_ACTIVE);
2326
2327         if (!(pf->pci_mem)) {
2328                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2329                 return (ENXIO);
2330         }
2331
2332         pf->osdep.mem_bus_space_tag =
2333                 rman_get_bustag(pf->pci_mem);
2334         pf->osdep.mem_bus_space_handle =
2335                 rman_get_bushandle(pf->pci_mem);
2336         pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2337         pf->osdep.flush_reg = I40E_GLGEN_STAT;
2338         pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2339
2340         pf->hw.back = &pf->osdep;
2341
2342         /*
2343         ** Now setup MSI or MSI/X, should
2344         ** return us the number of supported
2345         ** vectors. (Will be 1 for MSI)
2346         */
2347         pf->msix = ixl_init_msix(pf);
2348         return (0);
2349 }
2350
2351 static void
2352 ixl_free_pci_resources(struct ixl_pf * pf)
2353 {
2354         struct ixl_vsi          *vsi = &pf->vsi;
2355         struct ixl_queue        *que = vsi->queues;
2356         device_t                dev = pf->dev;
2357         int                     rid, memrid;
2358
2359         memrid = PCIR_BAR(IXL_BAR);
2360
2361         /* We may get here before stations are setup */
2362         if ((!ixl_enable_msix) || (que == NULL))
2363                 goto early;
2364
2365         /*
2366         **  Release all msix VSI resources:
2367         */
2368         for (int i = 0; i < vsi->num_queues; i++, que++) {
2369                 rid = que->msix + 1;
2370                 if (que->tag != NULL) {
2371                         bus_teardown_intr(dev, que->res, que->tag);
2372                         que->tag = NULL;
2373                 }
2374                 if (que->res != NULL)
2375                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2376         }
2377
2378 early:
2379         /* Clean the AdminQ interrupt last */
2380         if (pf->admvec) /* we are doing MSIX */
2381                 rid = pf->admvec + 1;
2382         else
2383                 (pf->msix != 0) ? (rid = 1):(rid = 0);
2384
2385         if (pf->tag != NULL) {
2386                 bus_teardown_intr(dev, pf->res, pf->tag);
2387                 pf->tag = NULL;
2388         }
2389         if (pf->res != NULL)
2390                 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2391
2392         if (pf->msix)
2393                 pci_release_msi(dev);
2394
2395         if (pf->msix_mem != NULL)
2396                 bus_release_resource(dev, SYS_RES_MEMORY,
2397                     memrid, pf->msix_mem);
2398
2399         if (pf->pci_mem != NULL)
2400                 bus_release_resource(dev, SYS_RES_MEMORY,
2401                     PCIR_BAR(0), pf->pci_mem);
2402
2403         return;
2404 }
2405
2406 static void
2407 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2408 {
2409         /* Display supported media types */
2410         if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2411                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2412
2413         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2414                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2415         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2416                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2417         if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2418                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2419
2420         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2421             phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4) ||
2422             phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR) ||
2423             phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2424             phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2425             phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2426             phy_type & (1 << I40E_PHY_TYPE_SFI) ||
2427             phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2428                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2429
2430         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2431                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2432         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2433                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2434         if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2435                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2436
2437         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2438             phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2439             phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2440             phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2441             phy_type & (1 << I40E_PHY_TYPE_XLPPI) ||
2442             /* KR4 uses CR4 until the OS has the real media type */
2443             phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2444                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2445
2446         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2447                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2448         if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2449                 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2450 }
2451
2452 /*********************************************************************
2453  *
2454  *  Setup networking device structure and register an interface.
2455  *
2456  **********************************************************************/
2457 static int
2458 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2459 {
2460         struct ifnet            *ifp;
2461         struct i40e_hw          *hw = vsi->hw;
2462         struct ixl_queue        *que = vsi->queues;
2463         struct i40e_aq_get_phy_abilities_resp abilities;
2464         enum i40e_status_code aq_error = 0;
2465
2466         INIT_DEBUGOUT("ixl_setup_interface: begin");
2467
2468         ifp = vsi->ifp = if_alloc(IFT_ETHER);
2469         if (ifp == NULL) {
2470                 device_printf(dev, "can not allocate ifnet structure\n");
2471                 return (-1);
2472         }
2473         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2474         ifp->if_mtu = ETHERMTU;
2475         ifp->if_baudrate = 4000000000;  // ??
2476         ifp->if_init = ixl_init;
2477         ifp->if_softc = vsi;
2478         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2479         ifp->if_ioctl = ixl_ioctl;
2480
2481 #if __FreeBSD_version >= 1100036
2482         if_setgetcounterfn(ifp, ixl_get_counter);
2483 #endif
2484
2485         ifp->if_transmit = ixl_mq_start;
2486
2487         ifp->if_qflush = ixl_qflush;
2488
2489         ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2490
2491         vsi->max_frame_size =
2492             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2493             + ETHER_VLAN_ENCAP_LEN;
2494
2495         /*
2496          * Tell the upper layer(s) we support long frames.
2497          */
2498         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2499
2500         ifp->if_capabilities |= IFCAP_HWCSUM;
2501         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2502         ifp->if_capabilities |= IFCAP_TSO;
2503         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2504         ifp->if_capabilities |= IFCAP_LRO;
2505
2506         /* VLAN capabilties */
2507         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2508                              |  IFCAP_VLAN_HWTSO
2509                              |  IFCAP_VLAN_MTU
2510                              |  IFCAP_VLAN_HWCSUM;
2511         ifp->if_capenable = ifp->if_capabilities;
2512
2513         /*
2514         ** Don't turn this on by default, if vlans are
2515         ** created on another pseudo device (eg. lagg)
2516         ** then vlan events are not passed thru, breaking
2517         ** operation, but with HW FILTER off it works. If
2518         ** using vlans directly on the ixl driver you can
2519         ** enable this and get full hardware tag filtering.
2520         */
2521         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2522
2523         /*
2524          * Specify the media types supported by this adapter and register
2525          * callbacks to update media and link information
2526          */
2527         ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2528                      ixl_media_status);
2529
2530         aq_error = i40e_aq_get_phy_capabilities(hw,
2531             FALSE, TRUE, &abilities, NULL);
2532         /* May need delay to detect fiber correctly */
2533         if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2534                 i40e_msec_delay(200);
2535                 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2536                     TRUE, &abilities, NULL);
2537         }
2538         if (aq_error) {
2539                 if (aq_error == I40E_ERR_UNKNOWN_PHY)
2540                         device_printf(dev, "Unknown PHY type detected!\n");
2541                 else
2542                         device_printf(dev,
2543                             "Error getting supported media types, err %d,"
2544                             " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2545                 return (0);
2546         }
2547
2548         ixl_add_ifmedia(vsi, abilities.phy_type);
2549
2550         /* Use autoselect media by default */
2551         ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2552         ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2553
2554         ether_ifattach(ifp, hw->mac.addr);
2555
2556         return (0);
2557 }
2558
2559 /*
2560 ** Run when the Admin Queue gets a
2561 ** link transition interrupt.
2562 */
2563 static void
2564 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
2565 {
2566         struct i40e_hw  *hw = &pf->hw;
2567         struct i40e_aqc_get_link_status *status =
2568             (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
2569         bool check;
2570
2571         hw->phy.get_link_info = TRUE;
2572         check = i40e_get_link_status(hw);
2573         pf->link_up = check;
2574 #ifdef IXL_DEBUG
2575         printf("Link is %s\n", check ? "up":"down");
2576 #endif
2577         /* Report if Unqualified modules are found */
2578         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
2579             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
2580             (!(status->link_info & I40E_AQ_LINK_UP)))
2581                 device_printf(pf->dev, "Link failed because "
2582                     "an unqualified module was detected\n");
2583
2584         return;
2585 }
2586
2587 /*********************************************************************
2588  *
2589  *  Get Firmware Switch configuration
2590  *      - this will need to be more robust when more complex
2591  *        switch configurations are enabled.
2592  *
2593  **********************************************************************/
2594 static int
2595 ixl_switch_config(struct ixl_pf *pf)
2596 {
2597         struct i40e_hw  *hw = &pf->hw;
2598         struct ixl_vsi  *vsi = &pf->vsi;
2599         device_t        dev = vsi->dev;
2600         struct i40e_aqc_get_switch_config_resp *sw_config;
2601         u8      aq_buf[I40E_AQ_LARGE_BUF];
2602         int     ret;
2603         u16     next = 0;
2604
2605         memset(&aq_buf, 0, sizeof(aq_buf));
2606         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
2607         ret = i40e_aq_get_switch_config(hw, sw_config,
2608             sizeof(aq_buf), &next, NULL);
2609         if (ret) {
2610                 device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n",
2611                     ret);
2612                 return (ret);
2613         }
2614 #ifdef IXL_DEBUG
2615         device_printf(dev,
2616             "Switch config: header reported: %d in structure, %d total\n",
2617             sw_config->header.num_reported, sw_config->header.num_total);
2618         for (int i = 0; i < sw_config->header.num_reported; i++) {
2619                 device_printf(dev,
2620                     "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
2621                     sw_config->element[i].element_type,
2622                     sw_config->element[i].seid,
2623                     sw_config->element[i].uplink_seid,
2624                     sw_config->element[i].downlink_seid);
2625         }
2626 #endif
2627         /* Simplified due to a single VSI at the moment */
2628         vsi->uplink_seid = sw_config->element[0].uplink_seid;
2629         vsi->downlink_seid = sw_config->element[0].downlink_seid;
2630         vsi->seid = sw_config->element[0].seid;
2631         return (ret);
2632 }
2633
2634 /*********************************************************************
2635  *
2636  *  Initialize the VSI:  this handles contexts, which means things
2637  *                       like the number of descriptors, buffer size,
2638  *                       plus we init the rings thru this function.
2639  *
2640  **********************************************************************/
2641 static int
2642 ixl_initialize_vsi(struct ixl_vsi *vsi)
2643 {
2644         struct ixl_pf           *pf = vsi->back;
2645         struct ixl_queue        *que = vsi->queues;
2646         device_t                dev = vsi->dev;
2647         struct i40e_hw          *hw = vsi->hw;
2648         struct i40e_vsi_context ctxt;
2649         int                     err = 0;
2650
2651         memset(&ctxt, 0, sizeof(ctxt));
2652         ctxt.seid = vsi->seid;
2653         if (pf->veb_seid != 0)
2654                 ctxt.uplink_seid = pf->veb_seid;
2655         ctxt.pf_num = hw->pf_id;
2656         err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2657         if (err) {
2658                 device_printf(dev,"get vsi params failed %x!!\n", err);
2659                 return (err);
2660         }
2661 #ifdef IXL_DEBUG
2662         printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
2663             "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
2664             "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
2665             ctxt.uplink_seid, ctxt.vsi_number,
2666             ctxt.vsis_allocated, ctxt.vsis_unallocated,
2667             ctxt.flags, ctxt.pf_num, ctxt.vf_num,
2668             ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
2669 #endif
2670         /*
2671         ** Set the queue and traffic class bits
2672         **  - when multiple traffic classes are supported
2673         **    this will need to be more robust.
2674         */
2675         ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2676         ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
2677         ctxt.info.queue_mapping[0] = 0;
2678         ctxt.info.tc_mapping[0] = 0x0800;
2679
2680         /* Set VLAN receive stripping mode */
2681         ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
2682         ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
2683         if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
2684             ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2685         else
2686             ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2687
2688         /* Keep copy of VSI info in VSI for statistic counters */
2689         memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2690
2691         /* Reset VSI statistics */
2692         ixl_vsi_reset_stats(vsi);
2693         vsi->hw_filters_add = 0;
2694         vsi->hw_filters_del = 0;
2695
2696         ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
2697
2698         err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2699         if (err) {
2700                 device_printf(dev,"update vsi params failed %x!!\n",
2701                    hw->aq.asq_last_status);
2702                 return (err);
2703         }
2704
2705         for (int i = 0; i < vsi->num_queues; i++, que++) {
2706                 struct tx_ring          *txr = &que->txr;
2707                 struct rx_ring          *rxr = &que->rxr;
2708                 struct i40e_hmc_obj_txq tctx;
2709                 struct i40e_hmc_obj_rxq rctx;
2710                 u32                     txctl;
2711                 u16                     size;
2712
2713
2714                 /* Setup the HMC TX Context  */
2715                 size = que->num_desc * sizeof(struct i40e_tx_desc);
2716                 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
2717                 tctx.new_context = 1;
2718                 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
2719                 tctx.qlen = que->num_desc;
2720                 tctx.fc_ena = 0;
2721                 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
2722                 /* Enable HEAD writeback */
2723                 tctx.head_wb_ena = 1;
2724                 tctx.head_wb_addr = txr->dma.pa +
2725                     (que->num_desc * sizeof(struct i40e_tx_desc));
2726                 tctx.rdylist_act = 0;
2727                 err = i40e_clear_lan_tx_queue_context(hw, i);
2728                 if (err) {
2729                         device_printf(dev, "Unable to clear TX context\n");
2730                         break;
2731                 }
2732                 err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
2733                 if (err) {
2734                         device_printf(dev, "Unable to set TX context\n");
2735                         break;
2736                 }
2737                 /* Associate the ring with this PF */
2738                 txctl = I40E_QTX_CTL_PF_QUEUE;
2739                 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2740                     I40E_QTX_CTL_PF_INDX_MASK);
2741                 wr32(hw, I40E_QTX_CTL(i), txctl);
2742                 ixl_flush(hw);
2743
2744                 /* Do ring (re)init */
2745                 ixl_init_tx_ring(que);
2746
2747                 /* Next setup the HMC RX Context  */
2748                 if (vsi->max_frame_size <= MCLBYTES)
2749                         rxr->mbuf_sz = MCLBYTES;
2750                 else
2751                         rxr->mbuf_sz = MJUMPAGESIZE;
2752
2753                 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
2754
2755                 /* Set up an RX context for the HMC */
2756                 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2757                 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
2758                 /* ignore header split for now */
2759                 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
2760                 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
2761                     vsi->max_frame_size : max_rxmax;
2762                 rctx.dtype = 0;
2763                 rctx.dsize = 1; /* do 32byte descriptors */
2764                 rctx.hsplit_0 = 0;  /* no HDR split initially */
2765                 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
2766                 rctx.qlen = que->num_desc;
2767                 rctx.tphrdesc_ena = 1;
2768                 rctx.tphwdesc_ena = 1;
2769                 rctx.tphdata_ena = 0;
2770                 rctx.tphhead_ena = 0;
2771                 rctx.lrxqthresh = 2;
2772                 rctx.crcstrip = 1;
2773                 rctx.l2tsel = 1;
2774                 rctx.showiv = 1;
2775                 rctx.fc_ena = 0;
2776                 rctx.prefena = 1;
2777
2778                 err = i40e_clear_lan_rx_queue_context(hw, i);
2779                 if (err) {
2780                         device_printf(dev,
2781                             "Unable to clear RX context %d\n", i);
2782                         break;
2783                 }
2784                 err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
2785                 if (err) {
2786                         device_printf(dev, "Unable to set RX context %d\n", i);
2787                         break;
2788                 }
2789                 err = ixl_init_rx_ring(que);
2790                 if (err) {
2791                         device_printf(dev, "Fail in init_rx_ring %d\n", i);
2792                         break;
2793                 }
2794                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), 0);
2795 #ifdef DEV_NETMAP
2796                 /* preserve queue */
2797                 if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
2798                         struct netmap_adapter *na = NA(vsi->ifp);
2799                         struct netmap_kring *kring = &na->rx_rings[i];
2800                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
2801                         wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
2802                 } else
2803 #endif /* DEV_NETMAP */
2804                 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
2805         }
2806         return (err);
2807 }
2808
2809
2810 /*********************************************************************
2811  *
2812  *  Free all VSI structs.
2813  *
2814  **********************************************************************/
2815 void
2816 ixl_free_vsi(struct ixl_vsi *vsi)
2817 {
2818         struct ixl_pf           *pf = (struct ixl_pf *)vsi->back;
2819         struct ixl_queue        *que = vsi->queues;
2820
2821         /* Free station queues */
2822         for (int i = 0; i < vsi->num_queues; i++, que++) {
2823                 struct tx_ring *txr = &que->txr;
2824                 struct rx_ring *rxr = &que->rxr;
2825
2826                 if (!mtx_initialized(&txr->mtx)) /* uninitialized */
2827                         continue;
2828                 IXL_TX_LOCK(txr);
2829                 ixl_free_que_tx(que);
2830                 if (txr->base)
2831                         i40e_free_dma_mem(&pf->hw, &txr->dma);
2832                 IXL_TX_UNLOCK(txr);
2833                 IXL_TX_LOCK_DESTROY(txr);
2834
2835                 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
2836                         continue;
2837                 IXL_RX_LOCK(rxr);
2838                 ixl_free_que_rx(que);
2839                 if (rxr->base)
2840                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
2841                 IXL_RX_UNLOCK(rxr);
2842                 IXL_RX_LOCK_DESTROY(rxr);
2843
2844         }
2845         free(vsi->queues, M_DEVBUF);
2846
2847         /* Free VSI filter list */
2848         ixl_free_mac_filters(vsi);
2849 }
2850
2851 static void
2852 ixl_free_mac_filters(struct ixl_vsi *vsi)
2853 {
2854         struct ixl_mac_filter *f;
2855
2856         while (!SLIST_EMPTY(&vsi->ftl)) {
2857                 f = SLIST_FIRST(&vsi->ftl);
2858                 SLIST_REMOVE_HEAD(&vsi->ftl, next);
2859                 free(f, M_DEVBUF);
2860         }
2861 }
2862
2863
2864 /*********************************************************************
2865  *
2866  *  Allocate memory for the VSI (virtual station interface) and their
2867  *  associated queues, rings and the descriptors associated with each,
2868  *  called only once at attach.
2869  *
2870  **********************************************************************/
2871 static int
2872 ixl_setup_stations(struct ixl_pf *pf)
2873 {
2874         device_t                dev = pf->dev;
2875         struct ixl_vsi          *vsi;
2876         struct ixl_queue        *que;
2877         struct tx_ring          *txr;
2878         struct rx_ring          *rxr;
2879         int                     rsize, tsize;
2880         int                     error = I40E_SUCCESS;
2881
2882         vsi = &pf->vsi;
2883         vsi->back = (void *)pf;
2884         vsi->hw = &pf->hw;
2885         vsi->id = 0;
2886         vsi->num_vlans = 0;
2887         vsi->back = pf;
2888
2889         /* Get memory for the station queues */
2890         if (!(vsi->queues =
2891             (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
2892             vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2893                 device_printf(dev, "Unable to allocate queue memory\n");
2894                 error = ENOMEM;
2895                 goto early;
2896         }
2897
2898         for (int i = 0; i < vsi->num_queues; i++) {
2899                 que = &vsi->queues[i];
2900                 que->num_desc = ixl_ringsz;
2901                 que->me = i;
2902                 que->vsi = vsi;
2903                 /* mark the queue as active */
2904                 vsi->active_queues |= (u64)1 << que->me;
2905                 txr = &que->txr;
2906                 txr->que = que;
2907                 txr->tail = I40E_QTX_TAIL(que->me);
2908
2909                 /* Initialize the TX lock */
2910                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2911                     device_get_nameunit(dev), que->me);
2912                 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
2913                 /* Create the TX descriptor ring */
2914                 tsize = roundup2((que->num_desc *
2915                     sizeof(struct i40e_tx_desc)) +
2916                     sizeof(u32), DBA_ALIGN);
2917                 if (i40e_allocate_dma_mem(&pf->hw,
2918                     &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
2919                         device_printf(dev,
2920                             "Unable to allocate TX Descriptor memory\n");
2921                         error = ENOMEM;
2922                         goto fail;
2923                 }
2924                 txr->base = (struct i40e_tx_desc *)txr->dma.va;
2925                 bzero((void *)txr->base, tsize);
2926                 /* Now allocate transmit soft structs for the ring */
2927                 if (ixl_allocate_tx_data(que)) {
2928                         device_printf(dev,
2929                             "Critical Failure setting up TX structures\n");
2930                         error = ENOMEM;
2931                         goto fail;
2932                 }
2933                 /* Allocate a buf ring */
2934                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
2935                     M_WAITOK, &txr->mtx);
2936                 if (txr->br == NULL) {
2937                         device_printf(dev,
2938                             "Critical Failure setting up TX buf ring\n");
2939                         error = ENOMEM;
2940                         goto fail;
2941                 }
2942
2943                 /*
2944                  * Next the RX queues...
2945                  */
2946                 rsize = roundup2(que->num_desc *
2947                     sizeof(union i40e_rx_desc), DBA_ALIGN);
2948                 rxr = &que->rxr;
2949                 rxr->que = que;
2950                 rxr->tail = I40E_QRX_TAIL(que->me);
2951
2952                 /* Initialize the RX side lock */
2953                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2954                     device_get_nameunit(dev), que->me);
2955                 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
2956
2957                 if (i40e_allocate_dma_mem(&pf->hw,
2958                     &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
2959                         device_printf(dev,
2960                             "Unable to allocate RX Descriptor memory\n");
2961                         error = ENOMEM;
2962                         goto fail;
2963                 }
2964                 rxr->base = (union i40e_rx_desc *)rxr->dma.va;
2965                 bzero((void *)rxr->base, rsize);
2966
2967                 /* Allocate receive soft structs for the ring*/
2968                 if (ixl_allocate_rx_data(que)) {
2969                         device_printf(dev,
2970                             "Critical Failure setting up receive structs\n");
2971                         error = ENOMEM;
2972                         goto fail;
2973                 }
2974         }
2975
2976         return (0);
2977
2978 fail:
2979         for (int i = 0; i < vsi->num_queues; i++) {
2980                 que = &vsi->queues[i];
2981                 rxr = &que->rxr;
2982                 txr = &que->txr;
2983                 if (rxr->base)
2984                         i40e_free_dma_mem(&pf->hw, &rxr->dma);
2985                 if (txr->base)
2986                         i40e_free_dma_mem(&pf->hw, &txr->dma);
2987         }
2988
2989 early:
2990         return (error);
2991 }
2992
2993 /*
2994 ** Provide a update to the queue RX
2995 ** interrupt moderation value.
2996 */
2997 static void
2998 ixl_set_queue_rx_itr(struct ixl_queue *que)
2999 {
3000         struct ixl_vsi  *vsi = que->vsi;
3001         struct i40e_hw  *hw = vsi->hw;
3002         struct rx_ring  *rxr = &que->rxr;
3003         u16             rx_itr;
3004         u16             rx_latency = 0;
3005         int             rx_bytes;
3006
3007
3008         /* Idle, do nothing */
3009         if (rxr->bytes == 0)
3010                 return;
3011
3012         if (ixl_dynamic_rx_itr) {
3013                 rx_bytes = rxr->bytes/rxr->itr;
3014                 rx_itr = rxr->itr;
3015
3016                 /* Adjust latency range */
3017                 switch (rxr->latency) {
3018                 case IXL_LOW_LATENCY:
3019                         if (rx_bytes > 10) {
3020                                 rx_latency = IXL_AVE_LATENCY;
3021                                 rx_itr = IXL_ITR_20K;
3022                         }
3023                         break;
3024                 case IXL_AVE_LATENCY:
3025                         if (rx_bytes > 20) {
3026                                 rx_latency = IXL_BULK_LATENCY;
3027                                 rx_itr = IXL_ITR_8K;
3028                         } else if (rx_bytes <= 10) {
3029                                 rx_latency = IXL_LOW_LATENCY;
3030                                 rx_itr = IXL_ITR_100K;
3031                         }
3032                         break;
3033                 case IXL_BULK_LATENCY:
3034                         if (rx_bytes <= 20) {
3035                                 rx_latency = IXL_AVE_LATENCY;
3036                                 rx_itr = IXL_ITR_20K;
3037                         }
3038                         break;
3039                  }
3040
3041                 rxr->latency = rx_latency;
3042
3043                 if (rx_itr != rxr->itr) {
3044                         /* do an exponential smoothing */
3045                         rx_itr = (10 * rx_itr * rxr->itr) /
3046                             ((9 * rx_itr) + rxr->itr);
3047                         rxr->itr = rx_itr & IXL_MAX_ITR;
3048                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3049                             que->me), rxr->itr);
3050                 }
3051         } else { /* We may have have toggled to non-dynamic */
3052                 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3053                         vsi->rx_itr_setting = ixl_rx_itr;
3054                 /* Update the hardware if needed */
3055                 if (rxr->itr != vsi->rx_itr_setting) {
3056                         rxr->itr = vsi->rx_itr_setting;
3057                         wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3058                             que->me), rxr->itr);
3059                 }
3060         }
3061         rxr->bytes = 0;
3062         rxr->packets = 0;
3063         return;
3064 }
3065
3066
3067 /*
3068 ** Provide a update to the queue TX
3069 ** interrupt moderation value.
3070 */
3071 static void
3072 ixl_set_queue_tx_itr(struct ixl_queue *que)
3073 {
3074         struct ixl_vsi  *vsi = que->vsi;
3075         struct i40e_hw  *hw = vsi->hw;
3076         struct tx_ring  *txr = &que->txr;
3077         u16             tx_itr;
3078         u16             tx_latency = 0;
3079         int             tx_bytes;
3080
3081
3082         /* Idle, do nothing */
3083         if (txr->bytes == 0)
3084                 return;
3085
3086         if (ixl_dynamic_tx_itr) {
3087                 tx_bytes = txr->bytes/txr->itr;
3088                 tx_itr = txr->itr;
3089
3090                 switch (txr->latency) {
3091                 case IXL_LOW_LATENCY:
3092                         if (tx_bytes > 10) {
3093                                 tx_latency = IXL_AVE_LATENCY;
3094                                 tx_itr = IXL_ITR_20K;
3095                         }
3096                         break;
3097                 case IXL_AVE_LATENCY:
3098                         if (tx_bytes > 20) {
3099                                 tx_latency = IXL_BULK_LATENCY;
3100                                 tx_itr = IXL_ITR_8K;
3101                         } else if (tx_bytes <= 10) {
3102                                 tx_latency = IXL_LOW_LATENCY;
3103                                 tx_itr = IXL_ITR_100K;
3104                         }
3105                         break;
3106                 case IXL_BULK_LATENCY:
3107                         if (tx_bytes <= 20) {
3108                                 tx_latency = IXL_AVE_LATENCY;
3109                                 tx_itr = IXL_ITR_20K;
3110                         }
3111                         break;
3112                 }
3113
3114                 txr->latency = tx_latency;
3115
3116                 if (tx_itr != txr->itr) {
3117                  /* do an exponential smoothing */
3118                         tx_itr = (10 * tx_itr * txr->itr) /
3119                             ((9 * tx_itr) + txr->itr);
3120                         txr->itr = tx_itr & IXL_MAX_ITR;
3121                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3122                             que->me), txr->itr);
3123                 }
3124
3125         } else { /* We may have have toggled to non-dynamic */
3126                 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3127                         vsi->tx_itr_setting = ixl_tx_itr;
3128                 /* Update the hardware if needed */
3129                 if (txr->itr != vsi->tx_itr_setting) {
3130                         txr->itr = vsi->tx_itr_setting;
3131                         wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3132                             que->me), txr->itr);
3133                 }
3134         }
3135         txr->bytes = 0;
3136         txr->packets = 0;
3137         return;
3138 }
3139
3140 #define QUEUE_NAME_LEN 32
3141
3142 static void
3143 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3144     struct sysctl_ctx_list *ctx, const char *sysctl_name)
3145 {
3146         struct sysctl_oid *tree;
3147         struct sysctl_oid_list *child;
3148         struct sysctl_oid_list *vsi_list;
3149
3150         tree = device_get_sysctl_tree(pf->dev);
3151         child = SYSCTL_CHILDREN(tree);
3152         vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3153                                    CTLFLAG_RD, NULL, "VSI Number");
3154         vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3155
3156         ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3157 }
3158
3159 static void
3160 ixl_add_hw_stats(struct ixl_pf *pf)
3161 {
3162         device_t dev = pf->dev;
3163         struct ixl_vsi *vsi = &pf->vsi;
3164         struct ixl_queue *queues = vsi->queues;
3165         struct i40e_hw_port_stats *pf_stats = &pf->stats;
3166
3167         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3168         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3169         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3170         struct sysctl_oid_list *vsi_list;
3171
3172         struct sysctl_oid *queue_node;
3173         struct sysctl_oid_list *queue_list;
3174
3175         struct tx_ring *txr;
3176         struct rx_ring *rxr;
3177         char queue_namebuf[QUEUE_NAME_LEN];
3178
3179         /* Driver statistics */
3180         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3181                         CTLFLAG_RD, &pf->watchdog_events,
3182                         "Watchdog timeouts");
3183         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3184                         CTLFLAG_RD, &pf->admin_irq,
3185                         "Admin Queue IRQ Handled");
3186
3187         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "vc_debug_level",
3188             CTLFLAG_RW, &pf->vc_debug_lvl, 0,
3189             "PF/VF Virtual Channel debug logging level");
3190
3191         ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3192         vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3193
3194         /* Queue statistics */
3195         for (int q = 0; q < vsi->num_queues; q++) {
3196                 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3197                 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3198                     OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3199                 queue_list = SYSCTL_CHILDREN(queue_node);
3200
3201                 txr = &(queues[q].txr);
3202                 rxr = &(queues[q].rxr);
3203
3204                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3205                                 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3206                                 "m_defrag() failed");
3207                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped",
3208                                 CTLFLAG_RD, &(queues[q].dropped_pkts),
3209                                 "Driver dropped packets");
3210                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3211                                 CTLFLAG_RD, &(queues[q].irqs),
3212                                 "irqs on this queue");
3213                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3214                                 CTLFLAG_RD, &(queues[q].tso),
3215                                 "TSO");
3216                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3217                                 CTLFLAG_RD, &(queues[q].tx_dma_setup),
3218                                 "Driver tx dma failure in xmit");
3219                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3220                                 CTLFLAG_RD, &(txr->no_desc),
3221                                 "Queue No Descriptor Available");
3222                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3223                                 CTLFLAG_RD, &(txr->total_packets),
3224                                 "Queue Packets Transmitted");
3225                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3226                                 CTLFLAG_RD, &(txr->tx_bytes),
3227                                 "Queue Bytes Transmitted");
3228                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3229                                 CTLFLAG_RD, &(rxr->rx_packets),
3230                                 "Queue Packets Received");
3231                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3232                                 CTLFLAG_RD, &(rxr->rx_bytes),
3233                                 "Queue Bytes Received");
3234         }
3235
3236         /* MAC stats */
3237         ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3238 }
3239
3240 static void
3241 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3242         struct sysctl_oid_list *child,
3243         struct i40e_eth_stats *eth_stats)
3244 {
3245         struct ixl_sysctl_info ctls[] =
3246         {
3247                 {&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3248                 {&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3249                         "Unicast Packets Received"},
3250                 {&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3251                         "Multicast Packets Received"},
3252                 {&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3253                         "Broadcast Packets Received"},
3254                 {&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3255                 {&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3256                 {&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3257                 {&eth_stats->tx_multicast, "mcast_pkts_txd",
3258                         "Multicast Packets Transmitted"},
3259                 {&eth_stats->tx_broadcast, "bcast_pkts_txd",
3260                         "Broadcast Packets Transmitted"},
3261                 // end
3262                 {0,0,0}
3263         };
3264
3265         struct ixl_sysctl_info *entry = ctls;
3266         while (entry->stat != 0)
3267         {
3268                 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3269                                 CTLFLAG_RD, entry->stat,
3270                                 entry->description);
3271                 entry++;
3272         }
3273 }
3274
3275 static void
3276 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3277         struct sysctl_oid_list *child,
3278         struct i40e_hw_port_stats *stats)
3279 {
3280         struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3281                                     CTLFLAG_RD, NULL, "Mac Statistics");
3282         struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3283
3284         struct i40e_eth_stats *eth_stats = &stats->eth;
3285         ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3286
3287         struct ixl_sysctl_info ctls[] =
3288         {
3289                 {&stats->crc_errors, "crc_errors", "CRC Errors"},
3290                 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3291                 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3292                 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3293                 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3294                 /* Packet Reception Stats */
3295                 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3296                 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3297                 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3298                 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3299                 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3300                 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3301                 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3302                 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3303                 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3304                 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3305                 {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3306                 {&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3307                 /* Packet Transmission Stats */
3308                 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3309                 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3310                 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3311                 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3312                 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3313                 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3314                 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3315                 /* Flow control */
3316                 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3317                 {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3318                 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3319                 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3320                 /* End */
3321                 {0,0,0}
3322         };
3323
3324         struct ixl_sysctl_info *entry = ctls;
3325         while (entry->stat != 0)
3326         {
3327                 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3328                                 CTLFLAG_RD, entry->stat,
3329                                 entry->description);
3330                 entry++;
3331         }
3332 }
3333
3334 /*
3335 ** ixl_config_rss - setup RSS
3336 **  - note this is done for the single vsi
3337 */
3338 static void ixl_config_rss(struct ixl_vsi *vsi)
3339 {
3340         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3341         struct i40e_hw  *hw = vsi->hw;
3342         u32             lut = 0;
3343         u64             set_hena = 0, hena;
3344         int             i, j, que_id;
3345 #ifdef RSS
3346         u32             rss_hash_config;
3347         u32             rss_seed[IXL_KEYSZ];
3348 #else
3349         u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3350                             0x183cfd8c, 0xce880440, 0x580cbc3c,
3351                             0x35897377, 0x328b25e1, 0x4fa98922,
3352                             0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3353 #endif
3354
3355 #ifdef RSS
3356         /* Fetch the configured RSS key */
3357         rss_getkey((uint8_t *) &rss_seed);
3358 #endif
3359
3360         /* Fill out hash function seed */
3361         for (i = 0; i < IXL_KEYSZ; i++)
3362                 wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3363
3364         /* Enable PCTYPES for RSS: */
3365 #ifdef RSS
3366         rss_hash_config = rss_gethashconfig();
3367         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3368                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3369         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3370                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3371         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3372                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3373         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3374                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3375         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3376                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3377         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3378                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3379         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3380                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3381 #else
3382         set_hena =
3383                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3384                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3385                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3386                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3387                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3388                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3389                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3390                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3391                 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3392                 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3393                 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3394 #endif
3395         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
3396             ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
3397         hena |= set_hena;
3398         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
3399         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3400
3401         /* Populate the LUT with max no. of queues in round robin fashion */
3402         for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3403                 if (j == vsi->num_queues)
3404                         j = 0;
3405 #ifdef RSS
3406                 /*
3407                  * Fetch the RSS bucket id for the given indirection entry.
3408                  * Cap it at the number of configured buckets (which is
3409                  * num_queues.)
3410                  */
3411                 que_id = rss_get_indirection_to_bucket(i);
3412                 que_id = que_id % vsi->num_queues;
3413 #else
3414                 que_id = j;
3415 #endif
3416                 /* lut = 4-byte sliding window of 4 lut entries */
3417                 lut = (lut << 8) | (que_id &
3418                     ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3419                 /* On i = 3, we have 4 entries in lut; write to the register */
3420                 if ((i & 3) == 3)
3421                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3422         }
3423         ixl_flush(hw);
3424 }
3425
3426
3427 /*
3428 ** This routine is run via an vlan config EVENT,
3429 ** it enables us to use the HW Filter table since
3430 ** we can get the vlan id. This just creates the
3431 ** entry in the soft version of the VFTA, init will
3432 ** repopulate the real table.
3433 */
3434 static void
3435 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3436 {
3437         struct ixl_vsi  *vsi = ifp->if_softc;
3438         struct i40e_hw  *hw = vsi->hw;
3439         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3440
3441         if (ifp->if_softc !=  arg)   /* Not our event */
3442                 return;
3443
3444         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3445                 return;
3446
3447         IXL_PF_LOCK(pf);
3448         ++vsi->num_vlans;
3449         ixl_add_filter(vsi, hw->mac.addr, vtag);
3450         IXL_PF_UNLOCK(pf);
3451 }
3452
3453 /*
3454 ** This routine is run via an vlan
3455 ** unconfig EVENT, remove our entry
3456 ** in the soft vfta.
3457 */
3458 static void
3459 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3460 {
3461         struct ixl_vsi  *vsi = ifp->if_softc;
3462         struct i40e_hw  *hw = vsi->hw;
3463         struct ixl_pf   *pf = (struct ixl_pf *)vsi->back;
3464
3465         if (ifp->if_softc !=  arg)
3466                 return;
3467
3468         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3469                 return;
3470
3471         IXL_PF_LOCK(pf);
3472         --vsi->num_vlans;
3473         ixl_del_filter(vsi, hw->mac.addr, vtag);
3474         IXL_PF_UNLOCK(pf);
3475 }
3476
3477 /*
3478 ** This routine updates vlan filters, called by init
3479 ** it scans the filter table and then updates the hw
3480 ** after a soft reset.
3481 */
3482 static void
3483 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
3484 {
3485         struct ixl_mac_filter   *f;
3486         int                     cnt = 0, flags;
3487
3488         if (vsi->num_vlans == 0)
3489                 return;
3490         /*
3491         ** Scan the filter list for vlan entries,
3492         ** mark them for addition and then call
3493         ** for the AQ update.
3494         */
3495         SLIST_FOREACH(f, &vsi->ftl, next) {
3496                 if (f->flags & IXL_FILTER_VLAN) {
3497                         f->flags |=
3498                             (IXL_FILTER_ADD |
3499                             IXL_FILTER_USED);
3500                         cnt++;
3501                 }
3502         }
3503         if (cnt == 0) {
3504                 printf("setup vlan: no filters found!\n");
3505                 return;
3506         }
3507         flags = IXL_FILTER_VLAN;
3508         flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3509         ixl_add_hw_filters(vsi, flags, cnt);
3510         return;
3511 }
3512
3513 /*
3514 ** Initialize filter list and add filters that the hardware
3515 ** needs to know about.
3516 */
3517 static void
3518 ixl_init_filters(struct ixl_vsi *vsi)
3519 {
3520         /* Add broadcast address */
3521         ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
3522 }
3523
3524 /*
3525 ** This routine adds mulicast filters
3526 */
3527 static void
3528 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
3529 {
3530         struct ixl_mac_filter *f;
3531
3532         /* Does one already exist */
3533         f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3534         if (f != NULL)
3535                 return;
3536
3537         f = ixl_get_filter(vsi);
3538         if (f == NULL) {
3539                 printf("WARNING: no filter available!!\n");
3540                 return;
3541         }
3542         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3543         f->vlan = IXL_VLAN_ANY;
3544         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
3545             | IXL_FILTER_MC);
3546
3547         return;
3548 }
3549
3550 static void
3551 ixl_reconfigure_filters(struct ixl_vsi *vsi)
3552 {
3553
3554         ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
3555 }
3556
3557 /*
3558 ** This routine adds macvlan filters
3559 */
3560 static void
3561 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3562 {
3563         struct ixl_mac_filter   *f, *tmp;
3564         struct ixl_pf           *pf;
3565         device_t                dev;
3566
3567         DEBUGOUT("ixl_add_filter: begin");
3568
3569         pf = vsi->back;
3570         dev = pf->dev;
3571
3572         /* Does one already exist */
3573         f = ixl_find_filter(vsi, macaddr, vlan);
3574         if (f != NULL)
3575                 return;
3576         /*
3577         ** Is this the first vlan being registered, if so we
3578         ** need to remove the ANY filter that indicates we are
3579         ** not in a vlan, and replace that with a 0 filter.
3580         */
3581         if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
3582                 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3583                 if (tmp != NULL) {
3584                         ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
3585                         ixl_add_filter(vsi, macaddr, 0);
3586                 }
3587         }
3588
3589         f = ixl_get_filter(vsi);
3590         if (f == NULL) {
3591                 device_printf(dev, "WARNING: no filter available!!\n");
3592                 return;
3593         }
3594         bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3595         f->vlan = vlan;
3596         f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3597         if (f->vlan != IXL_VLAN_ANY)
3598                 f->flags |= IXL_FILTER_VLAN;
3599         else
3600                 vsi->num_macs++;
3601
3602         ixl_add_hw_filters(vsi, f->flags, 1);
3603         return;
3604 }
3605
3606 static void
3607 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3608 {
3609         struct ixl_mac_filter *f;
3610
3611         f = ixl_find_filter(vsi, macaddr, vlan);
3612         if (f == NULL)
3613                 return;
3614
3615         f->flags |= IXL_FILTER_DEL;
3616         ixl_del_hw_filters(vsi, 1);
3617         vsi->num_macs--;
3618
3619         /* Check if this is the last vlan removal */
3620         if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
3621                 /* Switch back to a non-vlan filter */
3622                 ixl_del_filter(vsi, macaddr, 0);
3623                 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
3624         }
3625         return;
3626 }
3627
3628 /*
3629 ** Find the filter with both matching mac addr and vlan id
3630 */
3631 static struct ixl_mac_filter *
3632 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3633 {
3634         struct ixl_mac_filter   *f;
3635         bool                    match = FALSE;
3636
3637         SLIST_FOREACH(f, &vsi->ftl, next) {
3638                 if (!cmp_etheraddr(f->macaddr, macaddr))
3639                         continue;
3640                 if (f->vlan == vlan) {
3641                         match = TRUE;
3642                         break;
3643                 }
3644         }
3645
3646         if (!match)
3647                 f = NULL;
3648         return (f);
3649 }
3650
3651 /*
3652 ** This routine takes additions to the vsi filter
3653 ** table and creates an Admin Queue call to create
3654 ** the filters in the hardware.
3655 */
3656 static void
3657 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
3658 {
3659         struct i40e_aqc_add_macvlan_element_data *a, *b;
3660         struct ixl_mac_filter   *f;
3661         struct ixl_pf           *pf;
3662         struct i40e_hw          *hw;
3663         device_t                dev;
3664         int                     err, j = 0;
3665
3666         pf = vsi->back;
3667         dev = pf->dev;
3668         hw = &pf->hw;
3669         IXL_PF_LOCK_ASSERT(pf);
3670
3671         a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
3672             M_DEVBUF, M_NOWAIT | M_ZERO);
3673         if (a == NULL) {
3674                 device_printf(dev, "add_hw_filters failed to get memory\n");
3675                 return;
3676         }
3677
3678         /*
3679         ** Scan the filter list, each time we find one
3680         ** we add it to the admin queue array and turn off
3681         ** the add bit.
3682         */
3683         SLIST_FOREACH(f, &vsi->ftl, next) {
3684                 if (f->flags == flags) {
3685                         b = &a[j]; // a pox on fvl long names :)
3686                         bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
3687                         if (f->vlan == IXL_VLAN_ANY) {
3688                                 b->vlan_tag = 0;
3689                                 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
3690                         } else {
3691                                 b->vlan_tag = f->vlan;
3692                                 b->flags = 0;
3693                         }
3694                         b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
3695                         f->flags &= ~IXL_FILTER_ADD;
3696                         j++;
3697                 }
3698                 if (j == cnt)
3699                         break;
3700         }
3701         if (j > 0) {
3702                 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
3703                 if (err)
3704                         device_printf(dev, "aq_add_macvlan err %d, "
3705                             "aq_error %d\n", err, hw->aq.asq_last_status);
3706                 else
3707                         vsi->hw_filters_add += j;
3708         }
3709         free(a, M_DEVBUF);
3710         return;
3711 }
3712
3713 /*
3714 ** This routine takes removals in the vsi filter
3715 ** table and creates an Admin Queue call to delete
3716 ** the filters in the hardware.
3717 */
3718 static void
3719 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
3720 {
3721         struct i40e_aqc_remove_macvlan_element_data *d, *e;
3722         struct ixl_pf           *pf;
3723         struct i40e_hw          *hw;
3724         device_t                dev;
3725         struct ixl_mac_filter   *f, *f_temp;
3726         int                     err, j = 0;
3727
3728         DEBUGOUT("ixl_del_hw_filters: begin\n");
3729
3730         pf = vsi->back;
3731         hw = &pf->hw;
3732         dev = pf->dev;
3733
3734         d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
3735             M_DEVBUF, M_NOWAIT | M_ZERO);
3736         if (d == NULL) {
3737                 printf("del hw filter failed to get memory\n");
3738                 return;
3739         }
3740
3741         SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
3742                 if (f->flags & IXL_FILTER_DEL) {
3743                         e = &d[j]; // a pox on fvl long names :)
3744                         bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
3745                         e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
3746                         e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3747                         /* delete entry from vsi list */
3748                         SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
3749                         free(f, M_DEVBUF);
3750                         j++;
3751                 }
3752                 if (j == cnt)
3753                         break;
3754         }
3755         if (j > 0) {
3756                 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
3757                 /* NOTE: returns ENOENT every time but seems to work fine,
3758                    so we'll ignore that specific error. */
3759                 // TODO: Does this still occur on current firmwares?
3760                 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
3761                         int sc = 0;
3762                         for (int i = 0; i < j; i++)
3763                                 sc += (!d[i].error_code);
3764                         vsi->hw_filters_del += sc;
3765                         device_printf(dev,
3766                             "Failed to remove %d/%d filters, aq error %d\n",
3767                             j - sc, j, hw->aq.asq_last_status);
3768                 } else
3769                         vsi->hw_filters_del += j;
3770         }
3771         free(d, M_DEVBUF);
3772
3773         DEBUGOUT("ixl_del_hw_filters: end\n");
3774         return;
3775 }
3776
3777 static int
3778 ixl_enable_rings(struct ixl_vsi *vsi)
3779 {
3780         struct ixl_pf   *pf = vsi->back;
3781         struct i40e_hw  *hw = &pf->hw;
3782         int             index, error;
3783         u32             reg;
3784
3785         error = 0;
3786         for (int i = 0; i < vsi->num_queues; i++) {
3787                 index = vsi->first_queue + i;
3788                 i40e_pre_tx_queue_cfg(hw, index, TRUE);
3789
3790                 reg = rd32(hw, I40E_QTX_ENA(index));
3791                 reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3792                     I40E_QTX_ENA_QENA_STAT_MASK;
3793                 wr32(hw, I40E_QTX_ENA(index), reg);
3794                 /* Verify the enable took */
3795                 for (int j = 0; j < 10; j++) {
3796                         reg = rd32(hw, I40E_QTX_ENA(index));
3797                         if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
3798                                 break;
3799                         i40e_msec_delay(10);
3800                 }
3801                 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
3802                         device_printf(pf->dev, "TX queue %d disabled!\n",
3803                             index);
3804                         error = ETIMEDOUT;
3805                 }
3806
3807                 reg = rd32(hw, I40E_QRX_ENA(index));
3808                 reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3809                     I40E_QRX_ENA_QENA_STAT_MASK;
3810                 wr32(hw, I40E_QRX_ENA(index), reg);
3811                 /* Verify the enable took */
3812                 for (int j = 0; j < 10; j++) {
3813                         reg = rd32(hw, I40E_QRX_ENA(index));
3814                         if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3815                                 break;
3816                         i40e_msec_delay(10);
3817                 }
3818                 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
3819                         device_printf(pf->dev, "RX queue %d disabled!\n",
3820                             index);
3821                         error = ETIMEDOUT;
3822                 }
3823         }
3824
3825         return (error);
3826 }
3827
3828 static int
3829 ixl_disable_rings(struct ixl_vsi *vsi)
3830 {
3831         struct ixl_pf   *pf = vsi->back;
3832         struct i40e_hw  *hw = &pf->hw;
3833         int             index, error;
3834         u32             reg;
3835
3836         error = 0;
3837         for (int i = 0; i < vsi->num_queues; i++) {
3838                 index = vsi->first_queue + i;
3839
3840                 i40e_pre_tx_queue_cfg(hw, index, FALSE);
3841                 i40e_usec_delay(500);
3842
3843                 reg = rd32(hw, I40E_QTX_ENA(index));
3844                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3845                 wr32(hw, I40E_QTX_ENA(index), reg);
3846                 /* Verify the disable took */
3847                 for (int j = 0; j < 10; j++) {
3848                         reg = rd32(hw, I40E_QTX_ENA(index));
3849                         if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3850                                 break;
3851                         i40e_msec_delay(10);
3852                 }
3853                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
3854                         device_printf(pf->dev, "TX queue %d still enabled!\n",
3855                             index);
3856                         error = ETIMEDOUT;
3857                 }
3858
3859                 reg = rd32(hw, I40E_QRX_ENA(index));
3860                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3861                 wr32(hw, I40E_QRX_ENA(index), reg);
3862                 /* Verify the disable took */
3863                 for (int j = 0; j < 10; j++) {
3864                         reg = rd32(hw, I40E_QRX_ENA(index));
3865                         if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3866                                 break;
3867                         i40e_msec_delay(10);
3868                 }
3869                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
3870                         device_printf(pf->dev, "RX queue %d still enabled!\n",
3871                             index);
3872                         error = ETIMEDOUT;
3873                 }
3874         }
3875
3876         return (error);
3877 }
3878
3879 /**
3880  * ixl_handle_mdd_event
3881  *
3882  * Called from interrupt handler to identify possibly malicious vfs
3883  * (But also detects events from the PF, as well)
3884  **/
3885 static void ixl_handle_mdd_event(struct ixl_pf *pf)
3886 {
3887         struct i40e_hw *hw = &pf->hw;
3888         device_t dev = pf->dev;
3889         bool mdd_detected = false;
3890         bool pf_mdd_detected = false;
3891         u32 reg;
3892
3893         /* find what triggered the MDD event */
3894         reg = rd32(hw, I40E_GL_MDET_TX);
3895         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
3896                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
3897                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
3898                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
3899                                 I40E_GL_MDET_TX_EVENT_SHIFT;
3900                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
3901                                 I40E_GL_MDET_TX_QUEUE_SHIFT;
3902                 device_printf(dev,
3903                          "Malicious Driver Detection event 0x%02x"
3904                          " on TX queue %d pf number 0x%02x\n",
3905                          event, queue, pf_num);
3906                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
3907                 mdd_detected = true;
3908         }
3909         reg = rd32(hw, I40E_GL_MDET_RX);
3910         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
3911                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
3912                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
3913                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
3914                                 I40E_GL_MDET_RX_EVENT_SHIFT;
3915                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
3916                                 I40E_GL_MDET_RX_QUEUE_SHIFT;
3917                 device_printf(dev,
3918                          "Malicious Driver Detection event 0x%02x"
3919                          " on RX queue %d of function 0x%02x\n",
3920                          event, queue, func);
3921                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
3922                 mdd_detected = true;
3923         }
3924
3925         if (mdd_detected) {
3926                 reg = rd32(hw, I40E_PF_MDET_TX);
3927                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
3928                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
3929                         device_printf(dev,
3930                                  "MDD TX event is for this function 0x%08x",
3931                                  reg);
3932                         pf_mdd_detected = true;
3933                 }
3934                 reg = rd32(hw, I40E_PF_MDET_RX);
3935                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
3936                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
3937                         device_printf(dev,
3938                                  "MDD RX event is for this function 0x%08x",
3939                                  reg);
3940                         pf_mdd_detected = true;
3941                 }
3942         }
3943
3944         /* re-enable mdd interrupt cause */
3945         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
3946         reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3947         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
3948         ixl_flush(hw);
3949 }
3950
3951 static void
3952 ixl_enable_intr(struct ixl_vsi *vsi)
3953 {
3954         struct i40e_hw          *hw = vsi->hw;
3955         struct ixl_queue        *que = vsi->queues;
3956
3957         if (ixl_enable_msix) {
3958                 ixl_enable_adminq(hw);
3959                 for (int i = 0; i < vsi->num_queues; i++, que++)
3960                         ixl_enable_queue(hw, que->me);
3961         } else
3962                 ixl_enable_legacy(hw);
3963 }
3964
3965 static void
3966 ixl_disable_rings_intr(struct ixl_vsi *vsi)
3967 {
3968         struct i40e_hw          *hw = vsi->hw;
3969         struct ixl_queue        *que = vsi->queues;
3970
3971         for (int i = 0; i < vsi->num_queues; i++, que++)
3972                 ixl_disable_queue(hw, que->me);
3973 }
3974
3975 static void
3976 ixl_disable_intr(struct ixl_vsi *vsi)
3977 {
3978         struct i40e_hw          *hw = vsi->hw;
3979
3980         if (ixl_enable_msix)
3981                 ixl_disable_adminq(hw);
3982         else
3983                 ixl_disable_legacy(hw);
3984 }
3985
3986 static void
3987 ixl_enable_adminq(struct i40e_hw *hw)
3988 {
3989         u32             reg;
3990
3991         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
3992             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3993             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3994         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
3995         ixl_flush(hw);
3996         return;
3997 }
3998
3999 static void
4000 ixl_disable_adminq(struct i40e_hw *hw)
4001 {
4002         u32             reg;
4003
4004         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4005         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4006
4007         return;
4008 }
4009
4010 static void
4011 ixl_enable_queue(struct i40e_hw *hw, int id)
4012 {
4013         u32             reg;
4014
4015         reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4016             I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4017             (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4018         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4019 }
4020
4021 static void
4022 ixl_disable_queue(struct i40e_hw *hw, int id)
4023 {
4024         u32             reg;
4025
4026         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4027         wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4028
4029         return;
4030 }
4031
4032 static void
4033 ixl_enable_legacy(struct i40e_hw *hw)
4034 {
4035         u32             reg;
4036         reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4037             I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4038             (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4039         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4040 }
4041
4042 static void
4043 ixl_disable_legacy(struct i40e_hw *hw)
4044 {
4045         u32             reg;
4046
4047         reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4048         wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4049
4050         return;
4051 }
4052
4053 static void
4054 ixl_update_stats_counters(struct ixl_pf *pf)
4055 {
4056         struct i40e_hw  *hw = &pf->hw;
4057         struct ixl_vsi  *vsi = &pf->vsi;
4058         struct ixl_vf   *vf;
4059
4060         struct i40e_hw_port_stats *nsd = &pf->stats;
4061         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4062
4063         /* Update hw stats */
4064         ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4065                            pf->stat_offsets_loaded,
4066                            &osd->crc_errors, &nsd->crc_errors);
4067         ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4068                            pf->stat_offsets_loaded,
4069                            &osd->illegal_bytes, &nsd->illegal_bytes);
4070         ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4071                            I40E_GLPRT_GORCL(hw->port),
4072                            pf->stat_offsets_loaded,
4073                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4074         ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4075                            I40E_GLPRT_GOTCL(hw->port),
4076                            pf->stat_offsets_loaded,
4077                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4078         ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4079                            pf->stat_offsets_loaded,
4080                            &osd->eth.rx_discards,
4081                            &nsd->eth.rx_discards);
4082         ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4083                            I40E_GLPRT_UPRCL(hw->port),
4084                            pf->stat_offsets_loaded,
4085                            &osd->eth.rx_unicast,
4086                            &nsd->eth.rx_unicast);
4087         ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4088                            I40E_GLPRT_UPTCL(hw->port),
4089                            pf->stat_offsets_loaded,
4090                            &osd->eth.tx_unicast,
4091                            &nsd->eth.tx_unicast);
4092         ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4093                            I40E_GLPRT_MPRCL(hw->port),
4094                            pf->stat_offsets_loaded,
4095                            &osd->eth.rx_multicast,
4096                            &nsd->eth.rx_multicast);
4097         ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4098                            I40E_GLPRT_MPTCL(hw->port),
4099                            pf->stat_offsets_loaded,
4100                            &osd->eth.tx_multicast,
4101                            &nsd->eth.tx_multicast);
4102         ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4103                            I40E_GLPRT_BPRCL(hw->port),
4104                            pf->stat_offsets_loaded,
4105                            &osd->eth.rx_broadcast,
4106                            &nsd->eth.rx_broadcast);
4107         ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4108                            I40E_GLPRT_BPTCL(hw->port),
4109                            pf->stat_offsets_loaded,
4110                            &osd->eth.tx_broadcast,
4111                            &nsd->eth.tx_broadcast);
4112
4113         ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4114                            pf->stat_offsets_loaded,
4115                            &osd->tx_dropped_link_down,
4116                            &nsd->tx_dropped_link_down);
4117         ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4118                            pf->stat_offsets_loaded,
4119                            &osd->mac_local_faults,
4120                            &nsd->mac_local_faults);
4121         ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4122                            pf->stat_offsets_loaded,
4123                            &osd->mac_remote_faults,
4124                            &nsd->mac_remote_faults);
4125         ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4126                            pf->stat_offsets_loaded,
4127                            &osd->rx_length_errors,
4128                            &nsd->rx_length_errors);
4129
4130         /* Flow control (LFC) stats */
4131         ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4132                            pf->stat_offsets_loaded,
4133                            &osd->link_xon_rx, &nsd->link_xon_rx);
4134         ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4135                            pf->stat_offsets_loaded,
4136                            &osd->link_xon_tx, &nsd->link_xon_tx);
4137         ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4138                            pf->stat_offsets_loaded,
4139                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
4140         ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4141                            pf->stat_offsets_loaded,
4142                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
4143
4144         /* Packet size stats rx */
4145         ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4146                            I40E_GLPRT_PRC64L(hw->port),
4147                            pf->stat_offsets_loaded,
4148                            &osd->rx_size_64, &nsd->rx_size_64);
4149         ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4150                            I40E_GLPRT_PRC127L(hw->port),
4151                            pf->stat_offsets_loaded,
4152                            &osd->rx_size_127, &nsd->rx_size_127);
4153         ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4154                            I40E_GLPRT_PRC255L(hw->port),
4155                            pf->stat_offsets_loaded,
4156                            &osd->rx_size_255, &nsd->rx_size_255);
4157         ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4158                            I40E_GLPRT_PRC511L(hw->port),
4159                            pf->stat_offsets_loaded,
4160                            &osd->rx_size_511, &nsd->rx_size_511);
4161         ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4162                            I40E_GLPRT_PRC1023L(hw->port),
4163                            pf->stat_offsets_loaded,
4164                            &osd->rx_size_1023, &nsd->rx_size_1023);
4165         ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4166                            I40E_GLPRT_PRC1522L(hw->port),
4167                            pf->stat_offsets_loaded,
4168                            &osd->rx_size_1522, &nsd->rx_size_1522);
4169         ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4170                            I40E_GLPRT_PRC9522L(hw->port),
4171                            pf->stat_offsets_loaded,
4172                            &osd->rx_size_big, &nsd->rx_size_big);
4173
4174         /* Packet size stats tx */
4175         ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4176                            I40E_GLPRT_PTC64L(hw->port),
4177                            pf->stat_offsets_loaded,
4178                            &osd->tx_size_64, &nsd->tx_size_64);
4179         ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4180                            I40E_GLPRT_PTC127L(hw->port),
4181                            pf->stat_offsets_loaded,
4182                            &osd->tx_size_127, &nsd->tx_size_127);
4183         ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4184                            I40E_GLPRT_PTC255L(hw->port),
4185                            pf->stat_offsets_loaded,
4186                            &osd->tx_size_255, &nsd->tx_size_255);
4187         ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4188                            I40E_GLPRT_PTC511L(hw->port),
4189                            pf->stat_offsets_loaded,
4190                            &osd->tx_size_511, &nsd->tx_size_511);
4191         ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4192                            I40E_GLPRT_PTC1023L(hw->port),
4193                            pf->stat_offsets_loaded,
4194                            &osd->tx_size_1023, &nsd->tx_size_1023);
4195         ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4196                            I40E_GLPRT_PTC1522L(hw->port),
4197                            pf->stat_offsets_loaded,
4198                            &osd->tx_size_1522, &nsd->tx_size_1522);
4199         ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4200                            I40E_GLPRT_PTC9522L(hw->port),
4201                            pf->stat_offsets_loaded,
4202                            &osd->tx_size_big, &nsd->tx_size_big);
4203
4204         ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4205                            pf->stat_offsets_loaded,
4206                            &osd->rx_undersize, &nsd->rx_undersize);
4207         ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4208                            pf->stat_offsets_loaded,
4209                            &osd->rx_fragments, &nsd->rx_fragments);
4210         ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4211                            pf->stat_offsets_loaded,
4212                            &osd->rx_oversize, &nsd->rx_oversize);
4213         ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4214                            pf->stat_offsets_loaded,
4215                            &osd->rx_jabber, &nsd->rx_jabber);
4216         pf->stat_offsets_loaded = true;
4217         /* End hw stats */
4218
4219         /* Update vsi stats */
4220         ixl_update_vsi_stats(vsi);
4221
4222         for (int i = 0; i < pf->num_vfs; i++) {
4223                 vf = &pf->vfs[i];
4224                 if (vf->vf_flags & VF_FLAG_ENABLED)
4225                         ixl_update_eth_stats(&pf->vfs[i].vsi);
4226         }
4227 }
4228
4229 /*
4230 ** Tasklet handler for MSIX Adminq interrupts
4231 **  - do outside interrupt since it might sleep
4232 */
4233 static void
4234 ixl_do_adminq(void *context, int pending)
4235 {
4236         struct ixl_pf                   *pf = context;
4237         struct i40e_hw                  *hw = &pf->hw;
4238         struct ixl_vsi                  *vsi = &pf->vsi;
4239         struct i40e_arq_event_info      event;
4240         i40e_status                     ret;
4241         u32                             reg, loop = 0;
4242         u16                             opcode, result;
4243
4244         event.buf_len = IXL_AQ_BUF_SZ;
4245         event.msg_buf = malloc(event.buf_len,
4246             M_DEVBUF, M_NOWAIT | M_ZERO);
4247         if (!event.msg_buf) {
4248                 printf("Unable to allocate adminq memory\n");
4249                 return;
4250         }
4251
4252         IXL_PF_LOCK(pf);
4253         /* clean and process any events */
4254         do {
4255                 ret = i40e_clean_arq_element(hw, &event, &result);
4256                 if (ret)
4257                         break;
4258                 opcode = LE16_TO_CPU(event.desc.opcode);
4259                 switch (opcode) {
4260                 case i40e_aqc_opc_get_link_status:
4261                         ixl_link_event(pf, &event);
4262                         ixl_update_link_status(pf);
4263                         break;
4264                 case i40e_aqc_opc_send_msg_to_pf:
4265 #ifdef PCI_IOV
4266                         ixl_handle_vf_msg(pf, &event);
4267 #endif
4268                         break;
4269                 case i40e_aqc_opc_event_lan_overflow:
4270                         break;
4271                 default:
4272 #ifdef IXL_DEBUG
4273                         printf("AdminQ unknown event %x\n", opcode);
4274 #endif
4275                         break;
4276                 }
4277
4278         } while (result && (loop++ < IXL_ADM_LIMIT));
4279
4280         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4281         reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4282         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4283         free(event.msg_buf, M_DEVBUF);
4284
4285         /*
4286          * If there are still messages to process, reschedule ourselves.
4287          * Otherwise, re-enable our interrupt and go to sleep.
4288          */
4289         if (result > 0)
4290                 taskqueue_enqueue(pf->tq, &pf->adminq);
4291         else
4292                 ixl_enable_intr(vsi);
4293
4294         IXL_PF_UNLOCK(pf);
4295 }
4296
4297 static int
4298 ixl_debug_info(SYSCTL_HANDLER_ARGS)
4299 {
4300         struct ixl_pf   *pf;
4301         int             error, input = 0;
4302
4303         error = sysctl_handle_int(oidp, &input, 0, req);
4304
4305         if (error || !req->newptr)
4306                 return (error);
4307
4308         if (input == 1) {
4309                 pf = (struct ixl_pf *)arg1;
4310                 ixl_print_debug_info(pf);
4311         }
4312
4313         return (error);
4314 }
4315
4316 static void
4317 ixl_print_debug_info(struct ixl_pf *pf)
4318 {
4319         struct i40e_hw          *hw = &pf->hw;
4320         struct ixl_vsi          *vsi = &pf->vsi;
4321         struct ixl_queue        *que = vsi->queues;
4322         struct rx_ring          *rxr = &que->rxr;
4323         struct tx_ring          *txr = &que->txr;
4324         u32                     reg;
4325
4326
4327         printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
4328         printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
4329         printf("RX next check = %x\n", rxr->next_check);
4330         printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
4331         printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
4332         printf("TX desc avail = %x\n", txr->avail);
4333
4334         reg = rd32(hw, I40E_GLV_GORCL(0xc));
4335          printf("RX Bytes = %x\n", reg);
4336         reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
4337          printf("Port RX Bytes = %x\n", reg);
4338         reg = rd32(hw, I40E_GLV_RDPC(0xc));
4339          printf("RX discard = %x\n", reg);
4340         reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
4341          printf("Port RX discard = %x\n", reg);
4342
4343         reg = rd32(hw, I40E_GLV_TEPC(0xc));
4344          printf("TX errors = %x\n", reg);
4345         reg = rd32(hw, I40E_GLV_GOTCL(0xc));
4346          printf("TX Bytes = %x\n", reg);
4347
4348         reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
4349          printf("RX undersize = %x\n", reg);
4350         reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
4351          printf("RX fragments = %x\n", reg);
4352         reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
4353          printf("RX oversize = %x\n", reg);
4354         reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
4355          printf("RX length error = %x\n", reg);
4356         reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
4357          printf("mac remote fault = %x\n", reg);
4358         reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
4359          printf("mac local fault = %x\n", reg);
4360 }
4361
4362 /**
4363  * Update VSI-specific ethernet statistics counters.
4364  **/
4365 void ixl_update_eth_stats(struct ixl_vsi *vsi)
4366 {
4367         struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4368         struct i40e_hw *hw = &pf->hw;
4369         struct i40e_eth_stats *es;
4370         struct i40e_eth_stats *oes;
4371         struct i40e_hw_port_stats *nsd;
4372         u16 stat_idx = vsi->info.stat_counter_idx;
4373
4374         es = &vsi->eth_stats;
4375         oes = &vsi->eth_stats_offsets;
4376         nsd = &pf->stats;
4377
4378         /* Gather up the stats that the hw collects */
4379         ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4380                            vsi->stat_offsets_loaded,
4381                            &oes->tx_errors, &es->tx_errors);
4382         ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4383                            vsi->stat_offsets_loaded,
4384                            &oes->rx_discards, &es->rx_discards);
4385
4386         ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4387                            I40E_GLV_GORCL(stat_idx),
4388                            vsi->stat_offsets_loaded,
4389                            &oes->rx_bytes, &es->rx_bytes);
4390         ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4391                            I40E_GLV_UPRCL(stat_idx),
4392                            vsi->stat_offsets_loaded,
4393                            &oes->rx_unicast, &es->rx_unicast);
4394         ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4395                            I40E_GLV_MPRCL(stat_idx),
4396                            vsi->stat_offsets_loaded,
4397                            &oes->rx_multicast, &es->rx_multicast);
4398         ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4399                            I40E_GLV_BPRCL(stat_idx),
4400                            vsi->stat_offsets_loaded,
4401                            &oes->rx_broadcast, &es->rx_broadcast);
4402
4403         ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4404                            I40E_GLV_GOTCL(stat_idx),
4405                            vsi->stat_offsets_loaded,
4406                            &oes->tx_bytes, &es->tx_bytes);
4407         ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4408                            I40E_GLV_UPTCL(stat_idx),
4409                            vsi->stat_offsets_loaded,
4410                            &oes->tx_unicast, &es->tx_unicast);
4411         ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4412                            I40E_GLV_MPTCL(stat_idx),
4413                            vsi->stat_offsets_loaded,
4414                            &oes->tx_multicast, &es->tx_multicast);
4415         ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4416                            I40E_GLV_BPTCL(stat_idx),
4417                            vsi->stat_offsets_loaded,
4418                            &oes->tx_broadcast, &es->tx_broadcast);
4419         vsi->stat_offsets_loaded = true;
4420 }
4421
4422 static void
4423 ixl_update_vsi_stats(struct ixl_vsi *vsi)
4424 {
4425         struct ixl_pf           *pf;
4426         struct ifnet            *ifp;
4427         struct i40e_eth_stats   *es;
4428         u64                     tx_discards;
4429
4430         struct i40e_hw_port_stats *nsd;
4431
4432         pf = vsi->back;
4433         ifp = vsi->ifp;
4434         es = &vsi->eth_stats;
4435         nsd = &pf->stats;
4436
4437         ixl_update_eth_stats(vsi);
4438
4439         tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4440         for (int i = 0; i < vsi->num_queues; i++)
4441                 tx_discards += vsi->queues[i].txr.br->br_drops;
4442
4443         /* Update ifnet stats */
4444         IXL_SET_IPACKETS(vsi, es->rx_unicast +
4445                            es->rx_multicast +
4446                            es->rx_broadcast);
4447         IXL_SET_OPACKETS(vsi, es->tx_unicast +
4448                            es->tx_multicast +
4449                            es->tx_broadcast);
4450         IXL_SET_IBYTES(vsi, es->rx_bytes);
4451         IXL_SET_OBYTES(vsi, es->tx_bytes);
4452         IXL_SET_IMCASTS(vsi, es->rx_multicast);
4453         IXL_SET_OMCASTS(vsi, es->tx_multicast);
4454
4455         IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
4456             nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
4457             nsd->rx_jabber);
4458         IXL_SET_OERRORS(vsi, es->tx_errors);
4459         IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
4460         IXL_SET_OQDROPS(vsi, tx_discards);
4461         IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
4462         IXL_SET_COLLISIONS(vsi, 0);
4463 }
4464
4465 /**
4466  * Reset all of the stats for the given pf
4467  **/
4468 void ixl_pf_reset_stats(struct ixl_pf *pf)
4469 {
4470         bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
4471         bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
4472         pf->stat_offsets_loaded = false;
4473 }
4474
4475 /**
4476  * Resets all stats of the given vsi
4477  **/
4478 void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
4479 {
4480         bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
4481         bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
4482         vsi->stat_offsets_loaded = false;
4483 }
4484
4485 /**
4486  * Read and update a 48 bit stat from the hw
4487  *
4488  * Since the device stats are not reset at PFReset, they likely will not
4489  * be zeroed when the driver starts.  We'll save the first values read
4490  * and use them as offsets to be subtracted from the raw values in order
4491  * to report stats that count from zero.
4492  **/
4493 static void
4494 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
4495         bool offset_loaded, u64 *offset, u64 *stat)
4496 {
4497         u64 new_data;
4498
4499 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
4500         new_data = rd64(hw, loreg);
4501 #else
4502         /*
4503          * Use two rd32's instead of one rd64; FreeBSD versions before
4504          * 10 don't support 8 byte bus reads/writes.
4505          */
4506         new_data = rd32(hw, loreg);
4507         new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4508 #endif
4509
4510         if (!offset_loaded)
4511                 *offset = new_data;
4512         if (new_data >= *offset)
4513                 *stat = new_data - *offset;
4514         else
4515                 *stat = (new_data + ((u64)1 << 48)) - *offset;
4516         *stat &= 0xFFFFFFFFFFFFULL;
4517 }
4518
4519 /**
4520  * Read and update a 32 bit stat from the hw
4521  **/
4522 static void
4523 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
4524         bool offset_loaded, u64 *offset, u64 *stat)
4525 {
4526         u32 new_data;
4527
4528         new_data = rd32(hw, reg);
4529         if (!offset_loaded)
4530                 *offset = new_data;
4531         if (new_data >= *offset)
4532                 *stat = (u32)(new_data - *offset);
4533         else
4534                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
4535 }
4536
4537 /*
4538 ** Set flow control using sysctl:
4539 **      0 - off
4540 **      1 - rx pause
4541 **      2 - tx pause
4542 **      3 - full
4543 */
4544 static int
4545 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
4546 {
4547         /*
4548          * TODO: ensure flow control is disabled if
4549          * priority flow control is enabled
4550          *
4551          * TODO: ensure tx CRC by hardware should be enabled
4552          * if tx flow control is enabled.
4553          */
4554         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4555         struct i40e_hw *hw = &pf->hw;
4556         device_t dev = pf->dev;
4557         int error = 0;
4558         enum i40e_status_code aq_error = 0;
4559         u8 fc_aq_err = 0;
4560
4561         /* Get request */
4562         error = sysctl_handle_int(oidp, &pf->fc, 0, req);
4563         if ((error) || (req->newptr == NULL))
4564                 return (error);
4565         if (pf->fc < 0 || pf->fc > 3) {
4566                 device_printf(dev,
4567                     "Invalid fc mode; valid modes are 0 through 3\n");
4568                 return (EINVAL);
4569         }
4570
4571         /*
4572         ** Changing flow control mode currently does not work on
4573         ** 40GBASE-CR4 PHYs
4574         */
4575         if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4
4576             || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) {
4577                 device_printf(dev, "Changing flow control mode unsupported"
4578                     " on 40GBase-CR4 media.\n");
4579                 return (ENODEV);
4580         }
4581
4582         /* Set fc ability for port */
4583         hw->fc.requested_mode = pf->fc;
4584         aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
4585         if (aq_error) {
4586                 device_printf(dev,
4587                     "%s: Error setting new fc mode %d; fc_err %#x\n",
4588                     __func__, aq_error, fc_aq_err);
4589                 return (EAGAIN);
4590         }
4591
4592         return (0);
4593 }
4594
4595 static int
4596 ixl_current_speed(SYSCTL_HANDLER_ARGS)
4597 {
4598         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4599         struct i40e_hw *hw = &pf->hw;
4600         int error = 0, index = 0;
4601
4602         char *speeds[] = {
4603                 "Unknown",
4604                 "100M",
4605                 "1G",
4606                 "10G",
4607                 "40G",
4608                 "20G"
4609         };
4610
4611         ixl_update_link_status(pf);
4612
4613         switch (hw->phy.link_info.link_speed) {
4614         case I40E_LINK_SPEED_100MB:
4615                 index = 1;
4616                 break;
4617         case I40E_LINK_SPEED_1GB:
4618                 index = 2;
4619                 break;
4620         case I40E_LINK_SPEED_10GB:
4621                 index = 3;
4622                 break;
4623         case I40E_LINK_SPEED_40GB:
4624                 index = 4;
4625                 break;
4626         case I40E_LINK_SPEED_20GB:
4627                 index = 5;
4628                 break;
4629         case I40E_LINK_SPEED_UNKNOWN:
4630         default:
4631                 index = 0;
4632                 break;
4633         }
4634
4635         error = sysctl_handle_string(oidp, speeds[index],
4636             strlen(speeds[index]), req);
4637         return (error);
4638 }
4639
4640 static int
4641 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
4642 {
4643         struct i40e_hw *hw = &pf->hw;
4644         device_t dev = pf->dev;
4645         struct i40e_aq_get_phy_abilities_resp abilities;
4646         struct i40e_aq_set_phy_config config;
4647         enum i40e_status_code aq_error = 0;
4648
4649         /* Get current capability information */
4650         aq_error = i40e_aq_get_phy_capabilities(hw,
4651             FALSE, FALSE, &abilities, NULL);
4652         if (aq_error) {
4653                 device_printf(dev,
4654                     "%s: Error getting phy capabilities %d,"
4655                     " aq error: %d\n", __func__, aq_error,
4656                     hw->aq.asq_last_status);
4657                 return (EAGAIN);
4658         }
4659
4660         /* Prepare new config */
4661         bzero(&config, sizeof(config));
4662         config.phy_type = abilities.phy_type;
4663         config.abilities = abilities.abilities
4664             | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
4665         config.eee_capability = abilities.eee_capability;
4666         config.eeer = abilities.eeer_val;
4667         config.low_power_ctrl = abilities.d3_lpan;
4668         /* Translate into aq cmd link_speed */
4669         if (speeds & 0x8)
4670                 config.link_speed |= I40E_LINK_SPEED_20GB;
4671         if (speeds & 0x4)
4672                 config.link_speed |= I40E_LINK_SPEED_10GB;
4673         if (speeds & 0x2)
4674                 config.link_speed |= I40E_LINK_SPEED_1GB;
4675         if (speeds & 0x1)
4676                 config.link_speed |= I40E_LINK_SPEED_100MB;
4677
4678         /* Do aq command & restart link */
4679         aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
4680         if (aq_error) {
4681                 device_printf(dev,
4682                     "%s: Error setting new phy config %d,"
4683                     " aq error: %d\n", __func__, aq_error,
4684                     hw->aq.asq_last_status);
4685                 return (EAGAIN);
4686         }
4687
4688         /*
4689         ** This seems a bit heavy handed, but we
4690         ** need to get a reinit on some devices
4691         */
4692         IXL_PF_LOCK(pf);
4693         ixl_stop(pf);
4694         ixl_init_locked(pf);
4695         IXL_PF_UNLOCK(pf);
4696
4697         return (0);
4698 }
4699
4700 /*
4701 ** Control link advertise speed:
4702 **      Flags:
4703 **      0x1 - advertise 100 Mb
4704 **      0x2 - advertise 1G
4705 **      0x4 - advertise 10G
4706 **      0x8 - advertise 20G
4707 **
4708 ** Does not work on 40G devices.
4709 */
4710 static int
4711 ixl_set_advertise(SYSCTL_HANDLER_ARGS)
4712 {
4713         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4714         struct i40e_hw *hw = &pf->hw;
4715         device_t dev = pf->dev;
4716         int requested_ls = 0;
4717         int error = 0;
4718
4719         /*
4720         ** FW doesn't support changing advertised speed
4721         ** for 40G devices; speed is always 40G.
4722         */
4723         if (i40e_is_40G_device(hw->device_id))
4724                 return (ENODEV);
4725
4726         /* Read in new mode */
4727         requested_ls = pf->advertised_speed;
4728         error = sysctl_handle_int(oidp, &requested_ls, 0, req);
4729         if ((error) || (req->newptr == NULL))
4730                 return (error);
4731         /* Check for sane value */
4732         if (requested_ls < 0x1 || requested_ls > 0xE) {
4733                 device_printf(dev, "Invalid advertised speed; "
4734                     "valid modes are 0x1 through 0xE\n");
4735                 return (EINVAL);
4736         }
4737         /* Then check for validity based on adapter type */
4738         switch (hw->device_id) {
4739         case I40E_DEV_ID_10G_BASE_T:
4740                 if (requested_ls & 0x8) {
4741                         device_printf(dev,
4742                             "20Gbs speed not supported on this device.\n");
4743                         return (EINVAL);
4744                 }
4745                 break;
4746         case I40E_DEV_ID_20G_KR2:
4747                 if (requested_ls & 0x1) {
4748                         device_printf(dev,
4749                             "100Mbs speed not supported on this device.\n");
4750                         return (EINVAL);
4751                 }
4752                 break;
4753         default:
4754                 if (requested_ls & ~0x6) {
4755                         device_printf(dev,
4756                             "Only 1/10Gbs speeds are supported on this device.\n");
4757                         return (EINVAL);
4758                 }
4759                 break;
4760         }
4761
4762         /* Exit if no change */
4763         if (pf->advertised_speed == requested_ls)
4764                 return (0);
4765
4766         error = ixl_set_advertised_speeds(pf, requested_ls);
4767         if (error)
4768                 return (error);
4769
4770         pf->advertised_speed = requested_ls;
4771         ixl_update_link_status(pf);
4772         return (0);
4773 }
4774
4775 /*
4776 ** Get the width and transaction speed of
4777 ** the bus this adapter is plugged into.
4778 */
4779 static u16
4780 ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
4781 {
4782         u16                     link;
4783         u32                     offset;
4784
4785
4786         /* Get the PCI Express Capabilities offset */
4787         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4788
4789         /* ...and read the Link Status Register */
4790         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4791
4792         switch (link & I40E_PCI_LINK_WIDTH) {
4793         case I40E_PCI_LINK_WIDTH_1:
4794                 hw->bus.width = i40e_bus_width_pcie_x1;
4795                 break;
4796         case I40E_PCI_LINK_WIDTH_2:
4797                 hw->bus.width = i40e_bus_width_pcie_x2;
4798                 break;
4799         case I40E_PCI_LINK_WIDTH_4:
4800                 hw->bus.width = i40e_bus_width_pcie_x4;
4801                 break;
4802         case I40E_PCI_LINK_WIDTH_8:
4803                 hw->bus.width = i40e_bus_width_pcie_x8;
4804                 break;
4805         default:
4806                 hw->bus.width = i40e_bus_width_unknown;
4807                 break;
4808         }
4809
4810         switch (link & I40E_PCI_LINK_SPEED) {
4811         case I40E_PCI_LINK_SPEED_2500:
4812                 hw->bus.speed = i40e_bus_speed_2500;
4813                 break;
4814         case I40E_PCI_LINK_SPEED_5000:
4815                 hw->bus.speed = i40e_bus_speed_5000;
4816                 break;
4817         case I40E_PCI_LINK_SPEED_8000:
4818                 hw->bus.speed = i40e_bus_speed_8000;
4819                 break;
4820         default:
4821                 hw->bus.speed = i40e_bus_speed_unknown;
4822                 break;
4823         }
4824
4825
4826         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4827             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
4828             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
4829             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4830             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
4831             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
4832             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
4833             ("Unknown"));
4834
4835         if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
4836             (hw->bus.speed < i40e_bus_speed_8000)) {
4837                 device_printf(dev, "PCI-Express bandwidth available"
4838                     " for this device\n     may be insufficient for"
4839                     " optimal performance.\n");
4840                 device_printf(dev, "For expected performance a x8 "
4841                     "PCIE Gen3 slot is required.\n");
4842         }
4843
4844         return (link);
4845 }
4846
4847 static int
4848 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
4849 {
4850         struct ixl_pf   *pf = (struct ixl_pf *)arg1;
4851         struct i40e_hw  *hw = &pf->hw;
4852         char            buf[32];
4853
4854         snprintf(buf, sizeof(buf),
4855             "f%d.%d a%d.%d n%02x.%02x e%08x",
4856             hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
4857             hw->aq.api_maj_ver, hw->aq.api_min_ver,
4858             (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >>
4859             IXL_NVM_VERSION_HI_SHIFT,
4860             (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >>
4861             IXL_NVM_VERSION_LO_SHIFT,
4862             hw->nvm.eetrack);
4863         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4864 }
4865
4866
4867 #ifdef IXL_DEBUG_SYSCTL
4868 static int
4869 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
4870 {
4871         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4872         struct i40e_hw *hw = &pf->hw;
4873         struct i40e_link_status link_status;
4874         char buf[512];
4875
4876         enum i40e_status_code aq_error = 0;
4877
4878         aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
4879         if (aq_error) {
4880                 printf("i40e_aq_get_link_info() error %d\n", aq_error);
4881                 return (EPERM);
4882         }
4883
4884         sprintf(buf, "\n"
4885             "PHY Type : %#04x\n"
4886             "Speed    : %#04x\n"
4887             "Link info: %#04x\n"
4888             "AN info  : %#04x\n"
4889             "Ext info : %#04x",
4890             link_status.phy_type, link_status.link_speed,
4891             link_status.link_info, link_status.an_info,
4892             link_status.ext_info);
4893
4894         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4895 }
4896
4897 static int
4898 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
4899 {
4900         struct ixl_pf           *pf = (struct ixl_pf *)arg1;
4901         struct i40e_hw          *hw = &pf->hw;
4902         char                    buf[512];
4903         enum i40e_status_code   aq_error = 0;
4904
4905         struct i40e_aq_get_phy_abilities_resp abilities;
4906
4907         aq_error = i40e_aq_get_phy_capabilities(hw,
4908             TRUE, FALSE, &abilities, NULL);
4909         if (aq_error) {
4910                 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
4911                 return (EPERM);
4912         }
4913
4914         sprintf(buf, "\n"
4915             "PHY Type : %#010x\n"
4916             "Speed    : %#04x\n"
4917             "Abilities: %#04x\n"
4918             "EEE cap  : %#06x\n"
4919             "EEER reg : %#010x\n"
4920             "D3 Lpan  : %#04x",
4921             abilities.phy_type, abilities.link_speed,
4922             abilities.abilities, abilities.eee_capability,
4923             abilities.eeer_val, abilities.d3_lpan);
4924
4925         return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4926 }
4927
4928 static int
4929 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
4930 {
4931         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4932         struct ixl_vsi *vsi = &pf->vsi;
4933         struct ixl_mac_filter *f;
4934         char *buf, *buf_i;
4935
4936         int error = 0;
4937         int ftl_len = 0;
4938         int ftl_counter = 0;
4939         int buf_len = 0;
4940         int entry_len = 42;
4941
4942         SLIST_FOREACH(f, &vsi->ftl, next) {
4943                 ftl_len++;
4944         }
4945
4946         if (ftl_len < 1) {
4947                 sysctl_handle_string(oidp, "(none)", 6, req);
4948                 return (0);
4949         }
4950
4951         buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
4952         buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
4953
4954         sprintf(buf_i++, "\n");
4955         SLIST_FOREACH(f, &vsi->ftl, next) {
4956                 sprintf(buf_i,
4957                     MAC_FORMAT ", vlan %4d, flags %#06x",
4958                     MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
4959                 buf_i += entry_len;
4960                 /* don't print '\n' for last entry */
4961                 if (++ftl_counter != ftl_len) {
4962                         sprintf(buf_i, "\n");
4963                         buf_i++;
4964                 }
4965         }
4966
4967         error = sysctl_handle_string(oidp, buf, strlen(buf), req);
4968         if (error)
4969                 printf("sysctl error: %d\n", error);
4970         free(buf, M_DEVBUF);
4971         return error;
4972 }
4973
4974 #define IXL_SW_RES_SIZE 0x14
4975 static int
4976 ixl_res_alloc_cmp(const void *a, const void *b)
4977 {
4978         const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
4979         one = (struct i40e_aqc_switch_resource_alloc_element_resp *)a;
4980         two = (struct i40e_aqc_switch_resource_alloc_element_resp *)b;
4981
4982         return ((int)one->resource_type - (int)two->resource_type);
4983 }
4984
4985 static int
4986 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
4987 {
4988         struct ixl_pf *pf = (struct ixl_pf *)arg1;
4989         struct i40e_hw *hw = &pf->hw;
4990         device_t dev = pf->dev;
4991         struct sbuf *buf;
4992         int error = 0;
4993
4994         u8 num_entries;
4995         struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
4996
4997         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4998         if (!buf) {
4999                 device_printf(dev, "Could not allocate sbuf for output.\n");
5000                 return (ENOMEM);
5001         }
5002
5003         bzero(resp, sizeof(resp));
5004         error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5005                                 resp,
5006                                 IXL_SW_RES_SIZE,
5007                                 NULL);
5008         if (error) {
5009                 device_printf(dev,
5010                     "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5011                     __func__, error, hw->aq.asq_last_status);
5012                 sbuf_delete(buf);
5013                 return error;
5014         }
5015
5016         /* Sort entries by type for display */
5017         qsort(resp, num_entries,
5018             sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5019             &ixl_res_alloc_cmp);
5020
5021         sbuf_cat(buf, "\n");
5022         sbuf_printf(buf, "# of entries: %d\n", num_entries);
5023         sbuf_printf(buf,
5024             "Type | Guaranteed | Total | Used   | Un-allocated\n"
5025             "     | (this)     | (all) | (this) | (all)       \n");
5026         for (int i = 0; i < num_entries; i++) {
5027                 sbuf_printf(buf,
5028                     "%#4x | %10d   %5d   %6d   %12d",
5029                     resp[i].resource_type,
5030                     resp[i].guaranteed,
5031                     resp[i].total,
5032                     resp[i].used,
5033                     resp[i].total_unalloced);
5034                 if (i < num_entries - 1)
5035                         sbuf_cat(buf, "\n");
5036         }
5037
5038         error = sbuf_finish(buf);
5039         if (error) {
5040                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5041         }
5042         sbuf_delete(buf);
5043         return error;
5044 }
5045
5046 /*
5047 ** Caller must init and delete sbuf; this function will clear and
5048 ** finish it for caller.
5049 */
5050 static char *
5051 ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink)
5052 {
5053         sbuf_clear(s);
5054
5055         if (seid == 0 && uplink)
5056                 sbuf_cat(s, "Network");
5057         else if (seid == 0)
5058                 sbuf_cat(s, "Host");
5059         else if (seid == 1)
5060                 sbuf_cat(s, "EMP");
5061         else if (seid <= 5)
5062                 sbuf_printf(s, "MAC %d", seid - 2);
5063         else if (seid <= 15)
5064                 sbuf_cat(s, "Reserved");
5065         else if (seid <= 31)
5066                 sbuf_printf(s, "PF %d", seid - 16);
5067         else if (seid <= 159)
5068                 sbuf_printf(s, "VF %d", seid - 32);
5069         else if (seid <= 287)
5070                 sbuf_cat(s, "Reserved");
5071         else if (seid <= 511)
5072                 sbuf_cat(s, "Other"); // for other structures
5073         else if (seid <= 895)
5074                 sbuf_printf(s, "VSI %d", seid - 512);
5075         else if (seid <= 1023)
5076                 sbuf_printf(s, "Reserved");
5077         else
5078                 sbuf_cat(s, "Invalid");
5079
5080         sbuf_finish(s);
5081         return sbuf_data(s);
5082 }
5083
5084 static int
5085 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5086 {
5087         struct ixl_pf *pf = (struct ixl_pf *)arg1;
5088         struct i40e_hw *hw = &pf->hw;
5089         device_t dev = pf->dev;
5090         struct sbuf *buf;
5091         struct sbuf *nmbuf;
5092         int error = 0;
5093         u8 aq_buf[I40E_AQ_LARGE_BUF];
5094
5095         u16 next = 0;
5096         struct i40e_aqc_get_switch_config_resp *sw_config;
5097         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5098
5099         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5100         if (!buf) {
5101                 device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5102                 return (ENOMEM);
5103         }
5104
5105         error = i40e_aq_get_switch_config(hw, sw_config,
5106             sizeof(aq_buf), &next, NULL);
5107         if (error) {
5108                 device_printf(dev,
5109                     "%s: aq_get_switch_config() error %d, aq error %d\n",
5110                     __func__, error, hw->aq.asq_last_status);
5111                 sbuf_delete(buf);
5112                 return error;
5113         }
5114
5115         nmbuf = sbuf_new_auto();
5116         if (!nmbuf) {
5117                 device_printf(dev, "Could not allocate sbuf for name output.\n");
5118                 sbuf_delete(buf);
5119                 return (ENOMEM);
5120         }
5121
5122         sbuf_cat(buf, "\n");
5123         // Assuming <= 255 elements in switch
5124         sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported);
5125         /* Exclude:
5126         ** Revision -- all elements are revision 1 for now
5127         */
5128         sbuf_printf(buf,
5129             "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5130             "                |          |          | (uplink)\n");
5131         for (int i = 0; i < sw_config->header.num_reported; i++) {
5132                 // "%4d (%8s) | %8s   %8s   %#8x",
5133                 sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5134                 sbuf_cat(buf, " ");
5135                 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5136                     sw_config->element[i].seid, false));
5137                 sbuf_cat(buf, " | ");
5138                 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5139                     sw_config->element[i].uplink_seid, true));
5140                 sbuf_cat(buf, "   ");
5141                 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5142                     sw_config->element[i].downlink_seid, false));
5143                 sbuf_cat(buf, "   ");
5144                 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5145                 if (i < sw_config->header.num_reported - 1)
5146                         sbuf_cat(buf, "\n");
5147         }
5148         sbuf_delete(nmbuf);
5149
5150         error = sbuf_finish(buf);
5151         if (error) {
5152                 device_printf(dev, "Error finishing sbuf: %d\n", error);
5153         }
5154         sbuf_delete(buf);
5155
5156         return (error);
5157 }
5158 #endif /* IXL_DEBUG_SYSCTL */
5159
5160
5161 #ifdef PCI_IOV
5162 static int
5163 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5164 {
5165         struct i40e_hw *hw;
5166         struct ixl_vsi *vsi;
5167         struct i40e_vsi_context vsi_ctx;
5168         int i;
5169         uint16_t first_queue;
5170         enum i40e_status_code code;
5171
5172         hw = &pf->hw;
5173         vsi = &pf->vsi;
5174
5175         vsi_ctx.pf_num = hw->pf_id;
5176         vsi_ctx.uplink_seid = pf->veb_seid;
5177         vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
5178         vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5179         vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
5180
5181         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
5182
5183         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5184         vsi_ctx.info.switch_id = htole16(0);
5185
5186         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
5187         vsi_ctx.info.sec_flags = 0;
5188         if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
5189                 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
5190
5191         vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
5192         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
5193             I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5194
5195         vsi_ctx.info.valid_sections |=
5196             htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
5197         vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
5198         first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
5199         for (i = 0; i < IXLV_MAX_QUEUES; i++)
5200                 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
5201         for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
5202                 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
5203
5204         vsi_ctx.info.tc_mapping[0] = htole16(
5205             (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5206             (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
5207
5208         code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
5209         if (code != I40E_SUCCESS)
5210                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5211         vf->vsi.seid = vsi_ctx.seid;
5212         vf->vsi.vsi_num = vsi_ctx.vsi_number;
5213         vf->vsi.first_queue = first_queue;
5214         vf->vsi.num_queues = IXLV_MAX_QUEUES;
5215
5216         code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
5217         if (code != I40E_SUCCESS)
5218                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5219
5220         code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
5221         if (code != I40E_SUCCESS) {
5222                 device_printf(pf->dev, "Failed to disable BW limit: %d\n",
5223                     ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5224                 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5225         }
5226
5227         memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
5228         return (0);
5229 }
5230
5231 static int
5232 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5233 {
5234         struct i40e_hw *hw;
5235         int error;
5236
5237         hw = &pf->hw;
5238
5239         error = ixl_vf_alloc_vsi(pf, vf);
5240         if (error != 0)
5241                 return (error);
5242
5243         vf->vsi.hw_filters_add = 0;
5244         vf->vsi.hw_filters_del = 0;
5245         ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
5246         ixl_reconfigure_filters(&vf->vsi);
5247
5248         return (0);
5249 }
5250
5251 static void
5252 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
5253     uint32_t val)
5254 {
5255         uint32_t qtable;
5256         int index, shift;
5257
5258         /*
5259          * Two queues are mapped in a single register, so we have to do some
5260          * gymnastics to convert the queue number into a register index and
5261          * shift.
5262          */
5263         index = qnum / 2;
5264         shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
5265
5266         qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
5267         qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
5268         qtable |= val << shift;
5269         wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
5270 }
5271
5272 static void
5273 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
5274 {
5275         struct i40e_hw *hw;
5276         uint32_t qtable;
5277         int i;
5278
5279         hw = &pf->hw;
5280
5281         /*
5282          * Contiguous mappings aren't actually supported by the hardware,
5283          * so we have to use non-contiguous mappings.
5284          */
5285         wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
5286              I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
5287
5288         wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
5289             I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
5290
5291         for (i = 0; i < vf->vsi.num_queues; i++) {
5292                 qtable = (vf->vsi.first_queue + i) <<
5293                     I40E_VPLAN_QTABLE_QINDEX_SHIFT;
5294
5295                 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
5296         }
5297
5298         /* Map queues allocated to VF to its VSI. */
5299         for (i = 0; i < vf->vsi.num_queues; i++)
5300                 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
5301
5302         /* Set rest of VSI queues as unused. */
5303         for (; i < IXL_MAX_VSI_QUEUES; i++)
5304                 ixl_vf_map_vsi_queue(hw, vf, i,
5305                     I40E_VSILAN_QTABLE_QINDEX_0_MASK);
5306
5307         ixl_flush(hw);
5308 }
5309
5310 static void
5311 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
5312 {
5313         struct i40e_hw *hw;
5314
5315         hw = &pf->hw;
5316
5317         if (vsi->seid == 0)
5318                 return;
5319
5320         i40e_aq_delete_element(hw, vsi->seid, NULL);
5321 }
5322
5323 static void
5324 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
5325 {
5326
5327         wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
5328         ixl_flush(hw);
5329 }
5330
5331 static void
5332 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
5333 {
5334
5335         wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
5336             I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
5337         ixl_flush(hw);
5338 }
5339
5340 static void
5341 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
5342 {
5343         struct i40e_hw *hw;
5344         uint32_t vfint_reg, vpint_reg;
5345         int i;
5346
5347         hw = &pf->hw;
5348
5349         ixl_vf_vsi_release(pf, &vf->vsi);
5350
5351         /* Index 0 has a special register. */
5352         ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
5353
5354         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5355                 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
5356                 ixl_vf_disable_queue_intr(hw, vfint_reg);
5357         }
5358
5359         /* Index 0 has a special register. */
5360         ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
5361
5362         for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5363                 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
5364                 ixl_vf_unregister_intr(hw, vpint_reg);
5365         }
5366
5367         vf->vsi.num_queues = 0;
5368 }
5369
5370 static int
5371 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
5372 {
5373         struct i40e_hw *hw;
5374         int i;
5375         uint16_t global_vf_num;
5376         uint32_t ciad;
5377
5378         hw = &pf->hw;
5379         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5380
5381         wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
5382              (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
5383         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5384                 ciad = rd32(hw, I40E_PF_PCI_CIAD);
5385                 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
5386                         return (0);
5387                 DELAY(1);
5388         }
5389
5390         return (ETIMEDOUT);
5391 }
5392
5393 static void
5394 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5395 {
5396         struct i40e_hw *hw;
5397         uint32_t vfrtrig;
5398
5399         hw = &pf->hw;
5400
5401         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5402         vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5403         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5404         ixl_flush(hw);
5405
5406         ixl_reinit_vf(pf, vf);
5407 }
5408
5409 static void
5410 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5411 {
5412         struct i40e_hw *hw;
5413         uint32_t vfrstat, vfrtrig;
5414         int i, error;
5415
5416         hw = &pf->hw;
5417
5418         error = ixl_flush_pcie(pf, vf);
5419         if (error != 0)
5420                 device_printf(pf->dev,
5421                     "Timed out waiting for PCIe activity to stop on VF-%d\n",
5422                     vf->vf_num);
5423
5424         for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5425                 DELAY(10);
5426
5427                 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
5428                 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
5429                         break;
5430         }
5431
5432         if (i == IXL_VF_RESET_TIMEOUT)
5433                 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
5434
5435         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
5436
5437         vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5438         vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5439         wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5440
5441         if (vf->vsi.seid != 0)
5442                 ixl_disable_rings(&vf->vsi);
5443
5444         ixl_vf_release_resources(pf, vf);
5445         ixl_vf_setup_vsi(pf, vf);
5446         ixl_vf_map_queues(pf, vf);
5447
5448         wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
5449         ixl_flush(hw);
5450 }
5451
5452 static const char *
5453 ixl_vc_opcode_str(uint16_t op)
5454 {
5455
5456         switch (op) {
5457         case I40E_VIRTCHNL_OP_VERSION:
5458                 return ("VERSION");
5459         case I40E_VIRTCHNL_OP_RESET_VF:
5460                 return ("RESET_VF");
5461         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
5462                 return ("GET_VF_RESOURCES");
5463         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
5464                 return ("CONFIG_TX_QUEUE");
5465         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
5466                 return ("CONFIG_RX_QUEUE");
5467         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
5468                 return ("CONFIG_VSI_QUEUES");
5469         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
5470                 return ("CONFIG_IRQ_MAP");
5471         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
5472                 return ("ENABLE_QUEUES");
5473         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
5474                 return ("DISABLE_QUEUES");
5475         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
5476                 return ("ADD_ETHER_ADDRESS");
5477         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
5478                 return ("DEL_ETHER_ADDRESS");
5479         case I40E_VIRTCHNL_OP_ADD_VLAN:
5480                 return ("ADD_VLAN");
5481         case I40E_VIRTCHNL_OP_DEL_VLAN:
5482                 return ("DEL_VLAN");
5483         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
5484                 return ("CONFIG_PROMISCUOUS_MODE");
5485         case I40E_VIRTCHNL_OP_GET_STATS:
5486                 return ("GET_STATS");
5487         case I40E_VIRTCHNL_OP_FCOE:
5488                 return ("FCOE");
5489         case I40E_VIRTCHNL_OP_EVENT:
5490                 return ("EVENT");
5491         default:
5492                 return ("UNKNOWN");
5493         }
5494 }
5495
5496 static int
5497 ixl_vc_opcode_level(uint16_t opcode)
5498 {
5499
5500         switch (opcode) {
5501         case I40E_VIRTCHNL_OP_GET_STATS:
5502                 return (10);
5503         default:
5504                 return (5);
5505         }
5506 }
5507
5508 static void
5509 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5510     enum i40e_status_code status, void *msg, uint16_t len)
5511 {
5512         struct i40e_hw *hw;
5513         int global_vf_id;
5514
5515         hw = &pf->hw;
5516         global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
5517
5518         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
5519             "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
5520             ixl_vc_opcode_str(op), op, status, vf->vf_num);
5521
5522         i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
5523 }
5524
5525 static void
5526 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
5527 {
5528
5529         ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
5530 }
5531
5532 static void
5533 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5534     enum i40e_status_code status, const char *file, int line)
5535 {
5536
5537         I40E_VC_DEBUG(pf, 1,
5538             "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
5539             ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
5540         ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
5541 }
5542
5543 static void
5544 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5545     uint16_t msg_size)
5546 {
5547         struct i40e_virtchnl_version_info reply;
5548
5549         if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
5550                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
5551                     I40E_ERR_PARAM);
5552                 return;
5553         }
5554
5555         reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
5556         reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
5557         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
5558             sizeof(reply));
5559 }
5560
5561 static void
5562 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5563     uint16_t msg_size)
5564 {
5565
5566         if (msg_size != 0) {
5567                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
5568                     I40E_ERR_PARAM);
5569                 return;
5570         }
5571
5572         ixl_reset_vf(pf, vf);
5573
5574         /* No response to a reset message. */
5575 }
5576
5577 static void
5578 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5579     uint16_t msg_size)
5580 {
5581         struct i40e_virtchnl_vf_resource reply;
5582
5583         if (msg_size != 0) {
5584                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5585                     I40E_ERR_PARAM);
5586                 return;
5587         }
5588
5589         bzero(&reply, sizeof(reply));
5590
5591         reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2;
5592
5593         reply.num_vsis = 1;
5594         reply.num_queue_pairs = vf->vsi.num_queues;
5595         reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
5596         reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
5597         reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
5598         reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
5599         memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
5600
5601         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5602             I40E_SUCCESS, &reply, sizeof(reply));
5603 }
5604
5605 static int
5606 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5607     struct i40e_virtchnl_txq_info *info)
5608 {
5609         struct i40e_hw *hw;
5610         struct i40e_hmc_obj_txq txq;
5611         uint16_t global_queue_num, global_vf_num;
5612         enum i40e_status_code status;
5613         uint32_t qtx_ctl;
5614
5615         hw = &pf->hw;
5616         global_queue_num = vf->vsi.first_queue + info->queue_id;
5617         global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5618         bzero(&txq, sizeof(txq));
5619
5620         status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
5621         if (status != I40E_SUCCESS)
5622                 return (EINVAL);
5623
5624         txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
5625
5626         txq.head_wb_ena = info->headwb_enabled;
5627         txq.head_wb_addr = info->dma_headwb_addr;
5628         txq.qlen = info->ring_len;
5629         txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
5630         txq.rdylist_act = 0;
5631
5632         status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
5633         if (status != I40E_SUCCESS)
5634                 return (EINVAL);
5635
5636         qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
5637             (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
5638             (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
5639         wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
5640         ixl_flush(hw);
5641
5642         return (0);
5643 }
5644
5645 static int
5646 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5647     struct i40e_virtchnl_rxq_info *info)
5648 {
5649         struct i40e_hw *hw;
5650         struct i40e_hmc_obj_rxq rxq;
5651         uint16_t global_queue_num;
5652         enum i40e_status_code status;
5653
5654         hw = &pf->hw;
5655         global_queue_num = vf->vsi.first_queue + info->queue_id;
5656         bzero(&rxq, sizeof(rxq));
5657
5658         if (info->databuffer_size > IXL_VF_MAX_BUFFER)
5659                 return (EINVAL);
5660
5661         if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
5662             info->max_pkt_size < ETHER_MIN_LEN)
5663                 return (EINVAL);
5664
5665         if (info->splithdr_enabled) {
5666                 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
5667                         return (EINVAL);
5668
5669                 rxq.hsplit_0 = info->rx_split_pos &
5670                     (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
5671                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
5672                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
5673                      I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
5674                 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
5675
5676                 rxq.dtype = 2;
5677         }
5678
5679         status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
5680         if (status != I40E_SUCCESS)
5681                 return (EINVAL);
5682
5683         rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
5684         rxq.qlen = info->ring_len;
5685
5686         rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
5687
5688         rxq.dsize = 1;
5689         rxq.crcstrip = 1;
5690         rxq.l2tsel = 1;
5691
5692         rxq.rxmax = info->max_pkt_size;
5693         rxq.tphrdesc_ena = 1;
5694         rxq.tphwdesc_ena = 1;
5695         rxq.tphdata_ena = 1;
5696         rxq.tphhead_ena = 1;
5697         rxq.lrxqthresh = 2;
5698         rxq.prefena = 1;
5699
5700         status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
5701         if (status != I40E_SUCCESS)
5702                 return (EINVAL);
5703
5704         return (0);
5705 }
5706
5707 static void
5708 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5709     uint16_t msg_size)
5710 {
5711         struct i40e_virtchnl_vsi_queue_config_info *info;
5712         struct i40e_virtchnl_queue_pair_info *pair;
5713         int i;
5714
5715         if (msg_size < sizeof(*info)) {
5716                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5717                     I40E_ERR_PARAM);
5718                 return;
5719         }
5720
5721         info = msg;
5722         if (info->num_queue_pairs == 0) {
5723                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5724                     I40E_ERR_PARAM);
5725                 return;
5726         }
5727
5728         if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
5729                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5730                     I40E_ERR_PARAM);
5731                 return;
5732         }
5733
5734         if (info->vsi_id != vf->vsi.vsi_num) {
5735                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5736                     I40E_ERR_PARAM);
5737                 return;
5738         }
5739
5740         for (i = 0; i < info->num_queue_pairs; i++) {
5741                 pair = &info->qpair[i];
5742
5743                 if (pair->txq.vsi_id != vf->vsi.vsi_num ||
5744                     pair->rxq.vsi_id != vf->vsi.vsi_num ||
5745                     pair->txq.queue_id != pair->rxq.queue_id ||
5746                     pair->txq.queue_id >= vf->vsi.num_queues) {
5747
5748                         i40e_send_vf_nack(pf, vf,
5749                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5750                         return;
5751                 }
5752
5753                 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
5754                         i40e_send_vf_nack(pf, vf,
5755                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5756                         return;
5757                 }
5758
5759                 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
5760                         i40e_send_vf_nack(pf, vf,
5761                             I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5762                         return;
5763                 }
5764         }
5765
5766         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
5767 }
5768
5769 static void
5770 ixl_vf_set_qctl(struct ixl_pf *pf,
5771     const struct i40e_virtchnl_vector_map *vector,
5772     enum i40e_queue_type cur_type, uint16_t cur_queue,
5773     enum i40e_queue_type *last_type, uint16_t *last_queue)
5774 {
5775         uint32_t offset, qctl;
5776         uint16_t itr_indx;
5777
5778         if (cur_type == I40E_QUEUE_TYPE_RX) {
5779                 offset = I40E_QINT_RQCTL(cur_queue);
5780                 itr_indx = vector->rxitr_idx;
5781         } else {
5782                 offset = I40E_QINT_TQCTL(cur_queue);
5783                 itr_indx = vector->txitr_idx;
5784         }
5785
5786         qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
5787             (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
5788             (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
5789             I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5790             (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
5791
5792         wr32(&pf->hw, offset, qctl);
5793
5794         *last_type = cur_type;
5795         *last_queue = cur_queue;
5796 }
5797
5798 static void
5799 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
5800     const struct i40e_virtchnl_vector_map *vector)
5801 {
5802         struct i40e_hw *hw;
5803         u_int qindex;
5804         enum i40e_queue_type type, last_type;
5805         uint32_t lnklst_reg;
5806         uint16_t rxq_map, txq_map, cur_queue, last_queue;
5807
5808         hw = &pf->hw;
5809
5810         rxq_map = vector->rxq_map;
5811         txq_map = vector->txq_map;
5812
5813         last_queue = IXL_END_OF_INTR_LNKLST;
5814         last_type = I40E_QUEUE_TYPE_RX;
5815
5816         /*
5817          * The datasheet says to optimize performance, RX queues and TX queues
5818          * should be interleaved in the interrupt linked list, so we process
5819          * both at once here.
5820          */
5821         while ((rxq_map != 0) || (txq_map != 0)) {
5822                 if (txq_map != 0) {
5823                         qindex = ffs(txq_map) - 1;
5824                         type = I40E_QUEUE_TYPE_TX;
5825                         cur_queue = vf->vsi.first_queue + qindex;
5826                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
5827                             &last_type, &last_queue);
5828                         txq_map &= ~(1 << qindex);
5829                 }
5830
5831                 if (rxq_map != 0) {
5832                         qindex = ffs(rxq_map) - 1;
5833                         type = I40E_QUEUE_TYPE_RX;
5834                         cur_queue = vf->vsi.first_queue + qindex;
5835                         ixl_vf_set_qctl(pf, vector, type, cur_queue,
5836                             &last_type, &last_queue);
5837                         rxq_map &= ~(1 << qindex);
5838                 }
5839         }
5840
5841         if (vector->vector_id == 0)
5842                 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
5843         else
5844                 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
5845                     vf->vf_num);
5846         wr32(hw, lnklst_reg,
5847             (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
5848             (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
5849
5850         ixl_flush(hw);
5851 }
5852
5853 static void
5854 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5855     uint16_t msg_size)
5856 {
5857         struct i40e_virtchnl_irq_map_info *map;
5858         struct i40e_virtchnl_vector_map *vector;
5859         struct i40e_hw *hw;
5860         int i, largest_txq, largest_rxq;
5861
5862         hw = &pf->hw;
5863
5864         if (msg_size < sizeof(*map)) {
5865                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5866                     I40E_ERR_PARAM);
5867                 return;
5868         }
5869
5870         map = msg;
5871         if (map->num_vectors == 0) {
5872                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5873                     I40E_ERR_PARAM);
5874                 return;
5875         }
5876
5877         if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
5878                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5879                     I40E_ERR_PARAM);
5880                 return;
5881         }
5882
5883         for (i = 0; i < map->num_vectors; i++) {
5884                 vector = &map->vecmap[i];
5885
5886                 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
5887                     vector->vsi_id != vf->vsi.vsi_num) {
5888                         i40e_send_vf_nack(pf, vf,
5889                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
5890                         return;
5891                 }
5892
5893                 if (vector->rxq_map != 0) {
5894                         largest_rxq = fls(vector->rxq_map) - 1;
5895                         if (largest_rxq >= vf->vsi.num_queues) {
5896                                 i40e_send_vf_nack(pf, vf,
5897                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5898                                     I40E_ERR_PARAM);
5899                                 return;
5900                         }
5901                 }
5902
5903                 if (vector->txq_map != 0) {
5904                         largest_txq = fls(vector->txq_map) - 1;
5905                         if (largest_txq >= vf->vsi.num_queues) {
5906                                 i40e_send_vf_nack(pf, vf,
5907                                     I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5908                                     I40E_ERR_PARAM);
5909                                 return;
5910                         }
5911                 }
5912
5913                 if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
5914                     vector->txitr_idx > IXL_MAX_ITR_IDX) {
5915                         i40e_send_vf_nack(pf, vf,
5916                             I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5917                             I40E_ERR_PARAM);
5918                         return;
5919                 }
5920
5921                 ixl_vf_config_vector(pf, vf, vector);
5922         }
5923
5924         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
5925 }
5926
5927 static void
5928 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5929     uint16_t msg_size)
5930 {
5931         struct i40e_virtchnl_queue_select *select;
5932         int error;
5933
5934         if (msg_size != sizeof(*select)) {
5935                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5936                     I40E_ERR_PARAM);
5937                 return;
5938         }
5939
5940         select = msg;
5941         if (select->vsi_id != vf->vsi.vsi_num ||
5942             select->rx_queues == 0 || select->tx_queues == 0) {
5943                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5944                     I40E_ERR_PARAM);
5945                 return;
5946         }
5947
5948         error = ixl_enable_rings(&vf->vsi);
5949         if (error) {
5950                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
5951                     I40E_ERR_TIMEOUT);
5952                 return;
5953         }
5954
5955         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
5956 }
5957
5958 static void
5959 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
5960     void *msg, uint16_t msg_size)
5961 {
5962         struct i40e_virtchnl_queue_select *select;
5963         int error;
5964
5965         if (msg_size != sizeof(*select)) {
5966                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5967                     I40E_ERR_PARAM);
5968                 return;
5969         }
5970
5971         select = msg;
5972         if (select->vsi_id != vf->vsi.vsi_num ||
5973             select->rx_queues == 0 || select->tx_queues == 0) {
5974                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5975                     I40E_ERR_PARAM);
5976                 return;
5977         }
5978
5979         error = ixl_disable_rings(&vf->vsi);
5980         if (error) {
5981                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
5982                     I40E_ERR_TIMEOUT);
5983                 return;
5984         }
5985
5986         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
5987 }
5988
5989 static boolean_t
5990 ixl_zero_mac(const uint8_t *addr)
5991 {
5992         uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
5993
5994         return (cmp_etheraddr(addr, zero));
5995 }
5996
5997 static boolean_t
5998 ixl_bcast_mac(const uint8_t *addr)
5999 {
6000
6001         return (cmp_etheraddr(addr, ixl_bcast_addr));
6002 }
6003
6004 static int
6005 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6006 {
6007
6008         if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6009                 return (EINVAL);
6010
6011         /*
6012          * If the VF is not allowed to change its MAC address, don't let it
6013          * set a MAC filter for an address that is not a multicast address and
6014          * is not its assigned MAC.
6015          */
6016         if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6017             !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6018                 return (EPERM);
6019
6020         return (0);
6021 }
6022
6023 static void
6024 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6025     uint16_t msg_size)
6026 {
6027         struct i40e_virtchnl_ether_addr_list *addr_list;
6028         struct i40e_virtchnl_ether_addr *addr;
6029         struct ixl_vsi *vsi;
6030         int i;
6031         size_t expected_size;
6032
6033         vsi = &vf->vsi;
6034
6035         if (msg_size < sizeof(*addr_list)) {
6036                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6037                     I40E_ERR_PARAM);
6038                 return;
6039         }
6040
6041         addr_list = msg;
6042         expected_size = sizeof(*addr_list) +
6043             addr_list->num_elements * sizeof(*addr);
6044
6045         if (addr_list->num_elements == 0 ||
6046             addr_list->vsi_id != vsi->vsi_num ||
6047             msg_size != expected_size) {
6048                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6049                     I40E_ERR_PARAM);
6050                 return;
6051         }
6052
6053         for (i = 0; i < addr_list->num_elements; i++) {
6054                 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6055                         i40e_send_vf_nack(pf, vf,
6056                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6057                         return;
6058                 }
6059         }
6060
6061         for (i = 0; i < addr_list->num_elements; i++) {
6062                 addr = &addr_list->list[i];
6063                 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6064         }
6065
6066         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6067 }
6068
6069 static void
6070 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6071     uint16_t msg_size)
6072 {
6073         struct i40e_virtchnl_ether_addr_list *addr_list;
6074         struct i40e_virtchnl_ether_addr *addr;
6075         size_t expected_size;
6076         int i;
6077
6078         if (msg_size < sizeof(*addr_list)) {
6079                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6080                     I40E_ERR_PARAM);
6081                 return;
6082         }
6083
6084         addr_list = msg;
6085         expected_size = sizeof(*addr_list) +
6086             addr_list->num_elements * sizeof(*addr);
6087
6088         if (addr_list->num_elements == 0 ||
6089             addr_list->vsi_id != vf->vsi.vsi_num ||
6090             msg_size != expected_size) {
6091                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6092                     I40E_ERR_PARAM);
6093                 return;
6094         }
6095
6096         for (i = 0; i < addr_list->num_elements; i++) {
6097                 addr = &addr_list->list[i];
6098                 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6099                         i40e_send_vf_nack(pf, vf,
6100                             I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6101                         return;
6102                 }
6103         }
6104
6105         for (i = 0; i < addr_list->num_elements; i++) {
6106                 addr = &addr_list->list[i];
6107                 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6108         }
6109
6110         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6111 }
6112
6113 static enum i40e_status_code
6114 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6115 {
6116         struct i40e_vsi_context vsi_ctx;
6117
6118         vsi_ctx.seid = vf->vsi.seid;
6119
6120         bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6121         vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6122         vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6123             I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6124         return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6125 }
6126
6127 static void
6128 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6129     uint16_t msg_size)
6130 {
6131         struct i40e_virtchnl_vlan_filter_list *filter_list;
6132         enum i40e_status_code code;
6133         size_t expected_size;
6134         int i;
6135
6136         if (msg_size < sizeof(*filter_list)) {
6137                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6138                     I40E_ERR_PARAM);
6139                 return;
6140         }
6141
6142         filter_list = msg;
6143         expected_size = sizeof(*filter_list) +
6144             filter_list->num_elements * sizeof(uint16_t);
6145         if (filter_list->num_elements == 0 ||
6146             filter_list->vsi_id != vf->vsi.vsi_num ||
6147             msg_size != expected_size) {
6148                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6149                     I40E_ERR_PARAM);
6150                 return;
6151         }
6152
6153         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6154                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6155                     I40E_ERR_PARAM);
6156                 return;
6157         }
6158
6159         for (i = 0; i < filter_list->num_elements; i++) {
6160                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6161                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6162                             I40E_ERR_PARAM);
6163                         return;
6164                 }
6165         }
6166
6167         code = ixl_vf_enable_vlan_strip(pf, vf);
6168         if (code != I40E_SUCCESS) {
6169                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6170                     I40E_ERR_PARAM);
6171         }
6172
6173         for (i = 0; i < filter_list->num_elements; i++)
6174                 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6175
6176         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
6177 }
6178
6179 static void
6180 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6181     uint16_t msg_size)
6182 {
6183         struct i40e_virtchnl_vlan_filter_list *filter_list;
6184         int i;
6185         size_t expected_size;
6186
6187         if (msg_size < sizeof(*filter_list)) {
6188                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6189                     I40E_ERR_PARAM);
6190                 return;
6191         }
6192
6193         filter_list = msg;
6194         expected_size = sizeof(*filter_list) +
6195             filter_list->num_elements * sizeof(uint16_t);
6196         if (filter_list->num_elements == 0 ||
6197             filter_list->vsi_id != vf->vsi.vsi_num ||
6198             msg_size != expected_size) {
6199                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6200                     I40E_ERR_PARAM);
6201                 return;
6202         }
6203
6204         for (i = 0; i < filter_list->num_elements; i++) {
6205                 if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6206                         i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6207                             I40E_ERR_PARAM);
6208                         return;
6209                 }
6210         }
6211
6212         if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6213                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6214                     I40E_ERR_PARAM);
6215                 return;
6216         }
6217
6218         for (i = 0; i < filter_list->num_elements; i++)
6219                 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6220
6221         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
6222 }
6223
6224 static void
6225 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6226     void *msg, uint16_t msg_size)
6227 {
6228         struct i40e_virtchnl_promisc_info *info;
6229         enum i40e_status_code code;
6230
6231         if (msg_size != sizeof(*info)) {
6232                 i40e_send_vf_nack(pf, vf,
6233                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6234                 return;
6235         }
6236
6237         if (!vf->vf_flags & VF_FLAG_PROMISC_CAP) {
6238                 i40e_send_vf_nack(pf, vf,
6239                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6240                 return;
6241         }
6242
6243         info = msg;
6244         if (info->vsi_id != vf->vsi.vsi_num) {
6245                 i40e_send_vf_nack(pf, vf,
6246                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6247                 return;
6248         }
6249
6250         code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
6251             info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
6252         if (code != I40E_SUCCESS) {
6253                 i40e_send_vf_nack(pf, vf,
6254                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6255                 return;
6256         }
6257
6258         code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
6259             info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
6260         if (code != I40E_SUCCESS) {
6261                 i40e_send_vf_nack(pf, vf,
6262                     I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6263                 return;
6264         }
6265
6266         ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
6267 }
6268
6269 static void
6270 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6271     uint16_t msg_size)
6272 {
6273         struct i40e_virtchnl_queue_select *queue;
6274
6275         if (msg_size != sizeof(*queue)) {
6276                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6277                     I40E_ERR_PARAM);
6278                 return;
6279         }
6280
6281         queue = msg;
6282         if (queue->vsi_id != vf->vsi.vsi_num) {
6283                 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6284                     I40E_ERR_PARAM);
6285                 return;
6286         }
6287
6288         ixl_update_eth_stats(&vf->vsi);
6289
6290         ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6291             I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
6292 }
6293
6294 static void
6295 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
6296 {
6297         struct ixl_vf *vf;
6298         void *msg;
6299         uint16_t vf_num, msg_size;
6300         uint32_t opcode;
6301
6302         vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
6303         opcode = le32toh(event->desc.cookie_high);
6304
6305         if (vf_num >= pf->num_vfs) {
6306                 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
6307                 return;
6308         }
6309
6310         vf = &pf->vfs[vf_num];
6311         msg = event->msg_buf;
6312         msg_size = event->msg_len;
6313
6314         I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
6315             "Got msg %s(%d) from VF-%d of size %d\n",
6316             ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
6317
6318         switch (opcode) {
6319         case I40E_VIRTCHNL_OP_VERSION:
6320                 ixl_vf_version_msg(pf, vf, msg, msg_size);
6321                 break;
6322         case I40E_VIRTCHNL_OP_RESET_VF:
6323                 ixl_vf_reset_msg(pf, vf, msg, msg_size);
6324                 break;
6325         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6326                 ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
6327                 break;
6328         case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6329                 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
6330                 break;
6331         case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6332                 ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
6333                 break;
6334         case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6335                 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
6336                 break;
6337         case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6338                 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
6339                 break;
6340         case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6341                 ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
6342                 break;
6343         case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6344                 ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
6345                 break;
6346         case I40E_VIRTCHNL_OP_ADD_VLAN:
6347                 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
6348                 break;
6349         case I40E_VIRTCHNL_OP_DEL_VLAN:
6350                 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
6351                 break;
6352         case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6353                 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
6354                 break;
6355         case I40E_VIRTCHNL_OP_GET_STATS:
6356                 ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
6357                 break;
6358
6359         /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
6360         case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6361         case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6362         default:
6363                 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
6364                 break;
6365         }
6366 }
6367
6368 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
6369 static void
6370 ixl_handle_vflr(void *arg, int pending)
6371 {
6372         struct ixl_pf *pf;
6373         struct i40e_hw *hw;
6374         uint16_t global_vf_num;
6375         uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
6376         int i;
6377
6378         pf = arg;
6379         hw = &pf->hw;
6380
6381         IXL_PF_LOCK(pf);
6382         for (i = 0; i < pf->num_vfs; i++) {
6383                 global_vf_num = hw->func_caps.vf_base_id + i;
6384
6385                 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
6386                 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
6387                 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
6388                 if (vflrstat & vflrstat_mask) {
6389                         wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
6390                             vflrstat_mask);
6391
6392                         ixl_reinit_vf(pf, &pf->vfs[i]);
6393                 }
6394         }
6395
6396         icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
6397         icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
6398         wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
6399         ixl_flush(hw);
6400
6401         IXL_PF_UNLOCK(pf);
6402 }
6403
6404 static int
6405 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
6406 {
6407
6408         switch (err) {
6409         case I40E_AQ_RC_EPERM:
6410                 return (EPERM);
6411         case I40E_AQ_RC_ENOENT:
6412                 return (ENOENT);
6413         case I40E_AQ_RC_ESRCH:
6414                 return (ESRCH);
6415         case I40E_AQ_RC_EINTR:
6416                 return (EINTR);
6417         case I40E_AQ_RC_EIO:
6418                 return (EIO);
6419         case I40E_AQ_RC_ENXIO:
6420                 return (ENXIO);
6421         case I40E_AQ_RC_E2BIG:
6422                 return (E2BIG);
6423         case I40E_AQ_RC_EAGAIN:
6424                 return (EAGAIN);
6425         case I40E_AQ_RC_ENOMEM:
6426                 return (ENOMEM);
6427         case I40E_AQ_RC_EACCES:
6428                 return (EACCES);
6429         case I40E_AQ_RC_EFAULT:
6430                 return (EFAULT);
6431         case I40E_AQ_RC_EBUSY:
6432                 return (EBUSY);
6433         case I40E_AQ_RC_EEXIST:
6434                 return (EEXIST);
6435         case I40E_AQ_RC_EINVAL:
6436                 return (EINVAL);
6437         case I40E_AQ_RC_ENOTTY:
6438                 return (ENOTTY);
6439         case I40E_AQ_RC_ENOSPC:
6440                 return (ENOSPC);
6441         case I40E_AQ_RC_ENOSYS:
6442                 return (ENOSYS);
6443         case I40E_AQ_RC_ERANGE:
6444                 return (ERANGE);
6445         case I40E_AQ_RC_EFLUSHED:
6446                 return (EINVAL);        /* No exact equivalent in errno.h */
6447         case I40E_AQ_RC_BAD_ADDR:
6448                 return (EFAULT);
6449         case I40E_AQ_RC_EMODE:
6450                 return (EPERM);
6451         case I40E_AQ_RC_EFBIG:
6452                 return (EFBIG);
6453         default:
6454                 return (EINVAL);
6455         }
6456 }
6457
6458 static int
6459 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
6460 {
6461         struct ixl_pf *pf;
6462         struct i40e_hw *hw;
6463         struct ixl_vsi *pf_vsi;
6464         enum i40e_status_code ret;
6465         int i, error;
6466
6467         pf = device_get_softc(dev);
6468         hw = &pf->hw;
6469         pf_vsi = &pf->vsi;
6470
6471         IXL_PF_LOCK(pf);
6472         pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
6473             M_ZERO);
6474
6475         if (pf->vfs == NULL) {
6476                 error = ENOMEM;
6477                 goto fail;
6478         }
6479
6480         for (i = 0; i < num_vfs; i++)
6481                 sysctl_ctx_init(&pf->vfs[i].ctx);
6482
6483         ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
6484             1, FALSE, FALSE, &pf->veb_seid, NULL);
6485         if (ret != I40E_SUCCESS) {
6486                 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
6487                 device_printf(dev, "add_veb failed; code=%d error=%d", ret,
6488                     error);
6489                 goto fail;
6490         }
6491
6492         ixl_configure_msix(pf);
6493         ixl_enable_adminq(hw);
6494
6495         pf->num_vfs = num_vfs;
6496         IXL_PF_UNLOCK(pf);
6497         return (0);
6498
6499 fail:
6500         free(pf->vfs, M_IXL);
6501         pf->vfs = NULL;
6502         IXL_PF_UNLOCK(pf);
6503         return (error);
6504 }
6505
6506 static void
6507 ixl_iov_uninit(device_t dev)
6508 {
6509         struct ixl_pf *pf;
6510         struct i40e_hw *hw;
6511         struct ixl_vsi *vsi;
6512         struct ifnet *ifp;
6513         struct ixl_vf *vfs;
6514         int i, num_vfs;
6515
6516         pf = device_get_softc(dev);
6517         hw = &pf->hw;
6518         vsi = &pf->vsi;
6519         ifp = vsi->ifp;
6520
6521         IXL_PF_LOCK(pf);
6522         for (i = 0; i < pf->num_vfs; i++) {
6523                 if (pf->vfs[i].vsi.seid != 0)
6524                         i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
6525         }
6526
6527         if (pf->veb_seid != 0) {
6528                 i40e_aq_delete_element(hw, pf->veb_seid, NULL);
6529                 pf->veb_seid = 0;
6530         }
6531
6532         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
6533                 ixl_disable_intr(vsi);
6534
6535         vfs = pf->vfs;
6536         num_vfs = pf->num_vfs;
6537
6538         pf->vfs = NULL;
6539         pf->num_vfs = 0;
6540         IXL_PF_UNLOCK(pf);
6541
6542         /* Do this after the unlock as sysctl_ctx_free might sleep. */
6543         for (i = 0; i < num_vfs; i++)
6544                 sysctl_ctx_free(&vfs[i].ctx);
6545         free(vfs, M_IXL);
6546 }
6547
6548 static int
6549 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
6550 {
6551         char sysctl_name[QUEUE_NAME_LEN];
6552         struct ixl_pf *pf;
6553         struct ixl_vf *vf;
6554         const void *mac;
6555         size_t size;
6556         int error;
6557
6558         pf = device_get_softc(dev);
6559         vf = &pf->vfs[vfnum];
6560
6561         IXL_PF_LOCK(pf);
6562         vf->vf_num = vfnum;
6563
6564         vf->vsi.back = pf;
6565         vf->vf_flags = VF_FLAG_ENABLED;
6566         SLIST_INIT(&vf->vsi.ftl);
6567
6568         error = ixl_vf_setup_vsi(pf, vf);
6569         if (error != 0)
6570                 goto out;
6571
6572         if (nvlist_exists_binary(params, "mac-addr")) {
6573                 mac = nvlist_get_binary(params, "mac-addr", &size);
6574                 bcopy(mac, vf->mac, ETHER_ADDR_LEN);
6575
6576                 if (nvlist_get_bool(params, "allow-set-mac"))
6577                         vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6578         } else
6579                 /*
6580                  * If the administrator has not specified a MAC address then
6581                  * we must allow the VF to choose one.
6582                  */
6583                 vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6584
6585         if (nvlist_get_bool(params, "mac-anti-spoof"))
6586                 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
6587
6588         if (nvlist_get_bool(params, "allow-promisc"))
6589                 vf->vf_flags |= VF_FLAG_PROMISC_CAP;
6590
6591         vf->vf_flags |= VF_FLAG_VLAN_CAP;
6592
6593         ixl_reset_vf(pf, vf);
6594 out:
6595         IXL_PF_UNLOCK(pf);
6596         if (error == 0) {
6597                 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
6598                 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
6599         }
6600
6601         return (error);
6602 }
6603 #endif /* PCI_IOV */