sys/dev/ixgbe/if_ixv.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2017, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38
  39 #include "ixgbe.h"
  40 #include "ifdi_if.h"
  41
  42 #include <net/netmap.h>
  43 #include <dev/netmap/netmap_kern.h>
  44
  45 /************************************************************************
  46  * Driver version
  47  ************************************************************************/
  48 char ixv_driver_version[] = "2.0.1-k";
  49
  50 /************************************************************************
  51  * PCI Device ID Table
  52  *
  53  *   Used by probe to select devices to load on
  54  *   Last field stores an index into ixv_strings
  55  *   Last entry must be all 0s
  56  *
  57  *   { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  ************************************************************************/
  59 static pci_vendor_info_t ixv_vendor_info_array[] =
  60 {
  61         PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
  62         PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
  63         PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
  64         PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
  65         PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
  66         /* required last entry */
  67 PVID_END
  68 };
  69
  70 /************************************************************************
  71  * Function prototypes
  72  ************************************************************************/
  73 static void     *ixv_register(device_t dev);
  74 static int      ixv_if_attach_pre(if_ctx_t ctx);
  75 static int      ixv_if_attach_post(if_ctx_t ctx);
  76 static int      ixv_if_detach(if_ctx_t ctx);
  77
  78 static int      ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid);
  79 static int      ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
  80 static int      ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
  81 static void     ixv_if_queues_free(if_ctx_t ctx);
  82 static void     ixv_identify_hardware(if_ctx_t ctx);
  83 static void     ixv_init_device_features(struct adapter *);
  84 static int      ixv_allocate_pci_resources(if_ctx_t ctx);
  85 static void     ixv_free_pci_resources(if_ctx_t ctx);
  86 static int      ixv_setup_interface(if_ctx_t ctx);
  87 static void     ixv_if_media_status(if_ctx_t , struct ifmediareq *);
  88 static int      ixv_if_media_change(if_ctx_t ctx);
  89 static void     ixv_if_update_admin_status(if_ctx_t ctx);
  90 static int      ixv_if_msix_intr_assign(if_ctx_t ctx, int msix);
  91
  92 static int      ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
  93 static void     ixv_if_init(if_ctx_t ctx);
  94 static void     ixv_if_local_timer(if_ctx_t ctx, uint16_t qid);
  95 static void     ixv_if_stop(if_ctx_t ctx);
  96 static int      ixv_negotiate_api(struct adapter *);
  97
  98 static void     ixv_initialize_transmit_units(if_ctx_t ctx);
  99 static void     ixv_initialize_receive_units(if_ctx_t ctx);
 100 static void     ixv_initialize_rss_mapping(struct adapter *);
 101
 102 static void     ixv_setup_vlan_support(if_ctx_t ctx);
 103 static void     ixv_configure_ivars(struct adapter *);
 104 static void     ixv_if_enable_intr(if_ctx_t ctx);
 105 static void     ixv_if_disable_intr(if_ctx_t ctx);
 106 static void     ixv_if_multi_set(if_ctx_t ctx);
 107
 108 static void     ixv_if_register_vlan(if_ctx_t, u16);
 109 static void     ixv_if_unregister_vlan(if_ctx_t, u16);
 110
 111 static uint64_t ixv_if_get_counter(if_ctx_t, ift_counter);
 112
 113 static void     ixv_save_stats(struct adapter *);
 114 static void     ixv_init_stats(struct adapter *);
 115 static void     ixv_update_stats(struct adapter *);
 116 static void     ixv_add_stats_sysctls(struct adapter *adapter);
 117
 118 static int      ixv_sysctl_debug(SYSCTL_HANDLER_ARGS);
 119 static void     ixv_set_ivar(struct adapter *, u8, u8, s8);
 120
 121 static u8       *ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 122
 123 /* The MSI-X Interrupt handlers */
 124 static int      ixv_msix_que(void *);
 125 static int      ixv_msix_mbx(void *);
 126
 127 /************************************************************************
 128  * FreeBSD Device Interface Entry Points
 129  ************************************************************************/
 130 static device_method_t ixv_methods[] = {
 131         /* Device interface */
 132         DEVMETHOD(device_register, ixv_register),
 133         DEVMETHOD(device_probe, iflib_device_probe),
 134         DEVMETHOD(device_attach, iflib_device_attach),
 135         DEVMETHOD(device_detach, iflib_device_detach),
 136         DEVMETHOD(device_shutdown, iflib_device_shutdown),
 137         DEVMETHOD_END
 138 };
 139
 140 static driver_t ixv_driver = {
 141         "ixv", ixv_methods, sizeof(struct adapter),
 142 };
 143
 144 devclass_t ixv_devclass;
 145 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0);
 146 MODULE_DEPEND(ixv, pci, 1, 1, 1);
 147 MODULE_DEPEND(ixv, ether, 1, 1, 1);
 148 #ifdef DEV_NETMAP
 149 MODULE_DEPEND(ixv, netmap, 1, 1, 1);
 150 #endif /* DEV_NETMAP */
 151
 152 static device_method_t ixv_if_methods[] = {
 153         DEVMETHOD(ifdi_attach_pre, ixv_if_attach_pre),
 154         DEVMETHOD(ifdi_attach_post, ixv_if_attach_post),
 155         DEVMETHOD(ifdi_detach, ixv_if_detach),
 156         DEVMETHOD(ifdi_init, ixv_if_init),
 157         DEVMETHOD(ifdi_stop, ixv_if_stop),
 158         DEVMETHOD(ifdi_msix_intr_assign, ixv_if_msix_intr_assign),
 159         DEVMETHOD(ifdi_intr_enable, ixv_if_enable_intr),
 160         DEVMETHOD(ifdi_intr_disable, ixv_if_disable_intr),
 161         DEVMETHOD(ifdi_tx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
 162         DEVMETHOD(ifdi_rx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
 163         DEVMETHOD(ifdi_tx_queues_alloc, ixv_if_tx_queues_alloc),
 164         DEVMETHOD(ifdi_rx_queues_alloc, ixv_if_rx_queues_alloc),
 165         DEVMETHOD(ifdi_queues_free, ixv_if_queues_free),
 166         DEVMETHOD(ifdi_update_admin_status, ixv_if_update_admin_status),
 167         DEVMETHOD(ifdi_multi_set, ixv_if_multi_set),
 168         DEVMETHOD(ifdi_mtu_set, ixv_if_mtu_set),
 169         DEVMETHOD(ifdi_media_status, ixv_if_media_status),
 170         DEVMETHOD(ifdi_media_change, ixv_if_media_change),
 171         DEVMETHOD(ifdi_timer, ixv_if_local_timer),
 172         DEVMETHOD(ifdi_vlan_register, ixv_if_register_vlan),
 173         DEVMETHOD(ifdi_vlan_unregister, ixv_if_unregister_vlan),
 174         DEVMETHOD(ifdi_get_counter, ixv_if_get_counter),
 175         DEVMETHOD_END
 176 };
 177
 178 static driver_t ixv_if_driver = {
 179   "ixv_if", ixv_if_methods, sizeof(struct adapter)
 180 };
 181
 182 /*
 183  * TUNEABLE PARAMETERS:
 184  */
 185
 186 /* Flow control setting, default to full */
 187 static int ixv_flow_control = ixgbe_fc_full;
 188 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control);
 189
 190 /*
 191  * Header split: this causes the hardware to DMA
 192  * the header into a separate mbuf from the payload,
 193  * it can be a performance win in some workloads, but
 194  * in others it actually hurts, its off by default.
 195  */
 196 static int ixv_header_split = FALSE;
 197 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split);
 198
 199 /*
 200  * Shadow VFTA table, this is needed because
 201  * the real filter table gets cleared during
 202  * a soft reset and we need to repopulate it.
 203  */
 204 static u32 ixv_shadow_vfta[IXGBE_VFTA_SIZE];
 205 extern struct if_txrx ixgbe_txrx;
 206
 207 static struct if_shared_ctx ixv_sctx_init = {
 208         .isc_magic = IFLIB_MAGIC,
 209         .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
 210         .isc_tx_maxsize = IXGBE_TSO_SIZE,
 211
 212         .isc_tx_maxsegsize = PAGE_SIZE,
 213
 214         .isc_rx_maxsize = MJUM16BYTES,
 215         .isc_rx_nsegments = 1,
 216         .isc_rx_maxsegsize = MJUM16BYTES,
 217         .isc_nfl = 1,
 218         .isc_ntxqs = 1,
 219         .isc_nrxqs = 1,
 220         .isc_admin_intrcnt = 1,
 221         .isc_vendor_info = ixv_vendor_info_array,
 222         .isc_driver_version = ixv_driver_version,
 223         .isc_driver = &ixv_if_driver,
 224
 225         .isc_nrxd_min = {MIN_RXD},
 226         .isc_ntxd_min = {MIN_TXD},
 227         .isc_nrxd_max = {MAX_RXD},
 228         .isc_ntxd_max = {MAX_TXD},
 229         .isc_nrxd_default = {DEFAULT_RXD},
 230         .isc_ntxd_default = {DEFAULT_TXD},
 231 };
 232
 233 if_shared_ctx_t ixv_sctx = &ixv_sctx_init;
 234
 235 static void *
 236 ixv_register(device_t dev)
 237 {
 238         return (ixv_sctx);
 239 }
 240
 241 /************************************************************************
 242  * ixv_if_tx_queues_alloc
 243  ************************************************************************/
 244 static int
 245 ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
 246                        int ntxqs, int ntxqsets)
 247 {
 248         struct adapter     *adapter = iflib_get_softc(ctx);
 249         if_softc_ctx_t     scctx = adapter->shared;
 250         struct ix_tx_queue *que;
 251         int                i, j, error;
 252
 253         MPASS(adapter->num_tx_queues == ntxqsets);
 254         MPASS(ntxqs == 1);
 255
 256         /* Allocate queue structure memory */
 257         adapter->tx_queues =
 258             (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * ntxqsets,
 259                                          M_DEVBUF, M_NOWAIT | M_ZERO);
 260         if (!adapter->tx_queues) {
 261                 device_printf(iflib_get_dev(ctx),
 262                     "Unable to allocate TX ring memory\n");
 263                 return (ENOMEM);
 264         }
 265
 266         for (i = 0, que = adapter->tx_queues; i < ntxqsets; i++, que++) {
 267                 struct tx_ring *txr = &que->txr;
 268
 269                 txr->me = i;
 270                 txr->adapter =  que->adapter = adapter;
 271                 adapter->active_queues |= (u64)1 << txr->me;
 272
 273                 /* Allocate report status array */
 274                 if (!(txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) {
 275                         error = ENOMEM;
 276                         goto fail;
 277                 }
 278                 for (j = 0; j < scctx->isc_ntxd[0]; j++)
 279                         txr->tx_rsq[j] = QIDX_INVALID;
 280                 /* get the virtual and physical address of the hardware queues */
 281                 txr->tail = IXGBE_VFTDT(txr->me);
 282                 txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i*ntxqs];
 283                 txr->tx_paddr = paddrs[i*ntxqs];
 284
 285                 txr->bytes = 0;
 286                 txr->total_packets = 0;
 287
 288         }
 289
 290         device_printf(iflib_get_dev(ctx), "allocated for %d queues\n",
 291             adapter->num_tx_queues);
 292
 293         return (0);
 294
 295  fail:
 296         ixv_if_queues_free(ctx);
 297
 298         return (error);
 299 } /* ixv_if_tx_queues_alloc */
 300
 301 /************************************************************************
 302  * ixv_if_rx_queues_alloc
 303  ************************************************************************/
 304 static int
 305 ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
 306                        int nrxqs, int nrxqsets)
 307 {
 308         struct adapter     *adapter = iflib_get_softc(ctx);
 309         struct ix_rx_queue *que;
 310         int                i, error;
 311
 312         MPASS(adapter->num_rx_queues == nrxqsets);
 313         MPASS(nrxqs == 1);
 314
 315         /* Allocate queue structure memory */
 316         adapter->rx_queues =
 317             (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue) * nrxqsets,
 318                                          M_DEVBUF, M_NOWAIT | M_ZERO);
 319         if (!adapter->rx_queues) {
 320                 device_printf(iflib_get_dev(ctx),
 321                     "Unable to allocate TX ring memory\n");
 322                 error = ENOMEM;
 323                 goto fail;
 324         }
 325
 326         for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
 327                 struct rx_ring *rxr = &que->rxr;
 328                 rxr->me = i;
 329                 rxr->adapter = que->adapter = adapter;
 330
 331
 332                 /* get the virtual and physical address of the hw queues */
 333                 rxr->tail = IXGBE_VFRDT(rxr->me);
 334                 rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];
 335                 rxr->rx_paddr = paddrs[i*nrxqs];
 336                 rxr->bytes = 0;
 337                 rxr->que = que;
 338         }
 339
 340         device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n",
 341             adapter->num_rx_queues);
 342
 343         return (0);
 344
 345 fail:
 346         ixv_if_queues_free(ctx);
 347
 348         return (error);
 349 } /* ixv_if_rx_queues_alloc */
 350
 351 /************************************************************************
 352  * ixv_if_queues_free
 353  ************************************************************************/
 354 static void
 355 ixv_if_queues_free(if_ctx_t ctx)
 356 {
 357         struct adapter     *adapter = iflib_get_softc(ctx);
 358         struct ix_tx_queue *que = adapter->tx_queues;
 359         int                i;
 360
 361         if (que == NULL)
 362                 goto free;
 363
 364         for (i = 0; i < adapter->num_tx_queues; i++, que++) {
 365                 struct tx_ring *txr = &que->txr;
 366                 if (txr->tx_rsq == NULL)
 367                         break;
 368
 369                 free(txr->tx_rsq, M_DEVBUF);
 370                 txr->tx_rsq = NULL;
 371         }
 372         if (adapter->tx_queues != NULL)
 373                 free(adapter->tx_queues, M_DEVBUF);
 374 free:
 375         if (adapter->rx_queues != NULL)
 376                 free(adapter->rx_queues, M_DEVBUF);
 377         adapter->tx_queues = NULL;
 378         adapter->rx_queues = NULL;
 379 } /* ixv_if_queues_free */
 380
 381 /************************************************************************
 382  * ixv_if_attach_pre - Device initialization routine
 383  *
 384  *   Called when the driver is being loaded.
 385  *   Identifies the type of hardware, allocates all resources
 386  *   and initializes the hardware.
 387  *
 388  *   return 0 on success, positive on failure
 389  ************************************************************************/
 390 static int
 391 ixv_if_attach_pre(if_ctx_t ctx)
 392 {
 393         struct adapter  *adapter;
 394         device_t        dev;
 395         if_softc_ctx_t  scctx;
 396         struct ixgbe_hw *hw;
 397         int             error = 0;
 398
 399         INIT_DEBUGOUT("ixv_attach: begin");
 400
 401         /* Allocate, clear, and link in our adapter structure */
 402         dev = iflib_get_dev(ctx);
 403         adapter = iflib_get_softc(ctx);
 404         adapter->dev = dev;
 405         adapter->ctx = ctx;
 406         adapter->hw.back = adapter;
 407         scctx = adapter->shared = iflib_get_softc_ctx(ctx);
 408         adapter->media = iflib_get_media(ctx);
 409         hw = &adapter->hw;
 410
 411         /* Do base PCI setup - map BAR0 */
 412         if (ixv_allocate_pci_resources(ctx)) {
 413                 device_printf(dev, "ixv_allocate_pci_resources() failed!\n");
 414                 error = ENXIO;
 415                 goto err_out;
 416         }
 417
 418         /* SYSCTL APIs */
 419         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 420             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
 421             CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixv_sysctl_debug, "I",
 422             "Debug Info");
 423
 424         /* Determine hardware revision */
 425         ixv_identify_hardware(ctx);
 426         ixv_init_device_features(adapter);
 427
 428         /* Initialize the shared code */
 429         error = ixgbe_init_ops_vf(hw);
 430         if (error) {
 431                 device_printf(dev, "ixgbe_init_ops_vf() failed!\n");
 432                 error = EIO;
 433                 goto err_out;
 434         }
 435
 436         /* Setup the mailbox */
 437         ixgbe_init_mbx_params_vf(hw);
 438
 439         error = hw->mac.ops.reset_hw(hw);
 440         if (error == IXGBE_ERR_RESET_FAILED)
 441                 device_printf(dev, "...reset_hw() failure: Reset Failed!\n");
 442         else if (error)
 443                 device_printf(dev, "...reset_hw() failed with error %d\n",
 444                     error);
 445         if (error) {
 446                 error = EIO;
 447                 goto err_out;
 448         }
 449
 450         error = hw->mac.ops.init_hw(hw);
 451         if (error) {
 452                 device_printf(dev, "...init_hw() failed with error %d\n",
 453                     error);
 454                 error = EIO;
 455                 goto err_out;
 456         }
 457
 458         /* Negotiate mailbox API version */
 459         error = ixv_negotiate_api(adapter);
 460         if (error) {
 461                 device_printf(dev,
 462                     "Mailbox API negotiation failed during attach!\n");
 463                 goto err_out;
 464         }
 465
 466         /* If no mac address was assigned, make a random one */
 467         if (!ixv_check_ether_addr(hw->mac.addr)) {
 468                 u8 addr[ETHER_ADDR_LEN];
 469                 arc4rand(&addr, sizeof(addr), 0);
 470                 addr[0] &= 0xFE;
 471                 addr[0] |= 0x02;
 472                 bcopy(addr, hw->mac.addr, sizeof(addr));
 473                 bcopy(addr, hw->mac.perm_addr, sizeof(addr));
 474         }
 475
 476         /* Most of the iflib initialization... */
 477
 478         iflib_set_mac(ctx, hw->mac.addr);
 479         switch (adapter->hw.mac.type) {
 480         case ixgbe_mac_X550_vf:
 481         case ixgbe_mac_X550EM_x_vf:
 482         case ixgbe_mac_X550EM_a_vf:
 483                 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 2;
 484                 break;
 485         default:
 486                 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 1;
 487         }
 488         scctx->isc_txqsizes[0] =
 489             roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +
 490             sizeof(u32), DBA_ALIGN);
 491         scctx->isc_rxqsizes[0] =
 492             roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),
 493             DBA_ALIGN);
 494         /* XXX */
 495         scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
 496             CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO;
 497         scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;
 498         scctx->isc_msix_bar = PCIR_BAR(MSIX_82598_BAR);
 499         scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
 500         scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;
 501         scctx->isc_tx_tso_segsize_max = PAGE_SIZE;
 502
 503         scctx->isc_txrx = &ixgbe_txrx;
 504
 505         /*
 506          * Tell the upper layer(s) we support everything the PF
 507          * driver does except...
 508          *   hardware stats
 509          *   Wake-on-LAN
 510          */
 511         scctx->isc_capenable = IXGBE_CAPS;
 512         scctx->isc_capenable ^= IFCAP_HWSTATS | IFCAP_WOL;
 513
 514         INIT_DEBUGOUT("ixv_if_attach_pre: end");
 515
 516         return (0);
 517
 518 err_out:
 519         ixv_free_pci_resources(ctx);
 520
 521         return (error);
 522 } /* ixv_if_attach_pre */
 523
 524 static int
 525 ixv_if_attach_post(if_ctx_t ctx)
 526 {
 527         struct adapter *adapter = iflib_get_softc(ctx);
 528         device_t       dev = iflib_get_dev(ctx);
 529         int            error = 0;
 530
 531         /* Setup OS specific network interface */
 532         error = ixv_setup_interface(ctx);
 533         if (error) {
 534                 device_printf(dev, "Interface setup failed: %d\n", error);
 535                 goto end;
 536         }
 537
 538         /* Do the stats setup */
 539         ixv_save_stats(adapter);
 540         ixv_init_stats(adapter);
 541         ixv_add_stats_sysctls(adapter);
 542
 543 end:
 544         return error;
 545 } /* ixv_if_attach_post */
 546
 547 /************************************************************************
 548  * ixv_detach - Device removal routine
 549  *
 550  *   Called when the driver is being removed.
 551  *   Stops the adapter and deallocates all the resources
 552  *   that were allocated for driver operation.
 553  *
 554  *   return 0 on success, positive on failure
 555  ************************************************************************/
 556 static int
 557 ixv_if_detach(if_ctx_t ctx)
 558 {
 559         INIT_DEBUGOUT("ixv_detach: begin");
 560
 561         ixv_free_pci_resources(ctx);
 562
 563         return (0);
 564 } /* ixv_if_detach */
 565
 566 /************************************************************************
 567  * ixv_if_mtu_set
 568  ************************************************************************/
 569 static int
 570 ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
 571 {
 572         struct adapter *adapter = iflib_get_softc(ctx);
 573         struct ifnet   *ifp = iflib_get_ifp(ctx);
 574         int            error = 0;
 575
 576         IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 577         if (mtu > IXGBE_MAX_FRAME_SIZE - IXGBE_MTU_HDR) {
 578                 error = EINVAL;
 579         } else {
 580                 ifp->if_mtu = mtu;
 581                 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR;
 582         }
 583
 584         return error;
 585 } /* ixv_if_mtu_set */
 586
 587 /************************************************************************
 588  * ixv_if_init - Init entry point
 589  *
 590  *   Used in two ways: It is used by the stack as an init entry
 591  *   point in network interface structure. It is also used
 592  *   by the driver as a hw/sw initialization routine to get
 593  *   to a consistent state.
 594  *
 595  *   return 0 on success, positive on failure
 596  ************************************************************************/
 597 static void
 598 ixv_if_init(if_ctx_t ctx)
 599 {
 600         struct adapter  *adapter = iflib_get_softc(ctx);
 601         struct ifnet    *ifp = iflib_get_ifp(ctx);
 602         device_t        dev = iflib_get_dev(ctx);
 603         struct ixgbe_hw *hw = &adapter->hw;
 604         int             error = 0;
 605
 606         INIT_DEBUGOUT("ixv_if_init: begin");
 607         hw->adapter_stopped = FALSE;
 608         hw->mac.ops.stop_adapter(hw);
 609
 610         /* reprogram the RAR[0] in case user changed it. */
 611         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
 612
 613         /* Get the latest mac address, User can use a LAA */
 614         bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
 615         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1);
 616
 617         /* Reset VF and renegotiate mailbox API version */
 618         hw->mac.ops.reset_hw(hw);
 619         hw->mac.ops.start_hw(hw);
 620         error = ixv_negotiate_api(adapter);
 621         if (error) {
 622                 device_printf(dev,
 623                     "Mailbox API negotiation failed in if_init!\n");
 624                 return;
 625         }
 626
 627         ixv_initialize_transmit_units(ctx);
 628
 629         /* Setup Multicast table */
 630         ixv_if_multi_set(ctx);
 631
 632         /*
 633          * Determine the correct mbuf pool
 634          * for doing jumbo/headersplit
 635          */
 636         if (ifp->if_mtu > ETHERMTU)
 637                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 638         else
 639                 adapter->rx_mbuf_sz = MCLBYTES;
 640
 641         /* Configure RX settings */
 642         ixv_initialize_receive_units(ctx);
 643
 644         /* Set up VLAN offload and filter */
 645         ixv_setup_vlan_support(ctx);
 646
 647         /* Set up MSI-X routing */
 648         ixv_configure_ivars(adapter);
 649
 650         /* Set up auto-mask */
 651         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE);
 652
 653         /* Set moderation on the Link interrupt */
 654         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->vector), IXGBE_LINK_ITR);
 655
 656         /* Stats init */
 657         ixv_init_stats(adapter);
 658
 659         /* Config/Enable Link */
 660         hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up,
 661             FALSE);
 662
 663         /* And now turn on interrupts */
 664         ixv_if_enable_intr(ctx);
 665
 666         return;
 667 } /* ixv_if_init */
 668
 669 /************************************************************************
 670  * ixv_enable_queue
 671  ************************************************************************/
 672 static inline void
 673 ixv_enable_queue(struct adapter *adapter, u32 vector)
 674 {
 675         struct ixgbe_hw *hw = &adapter->hw;
 676         u32             queue = 1 << vector;
 677         u32             mask;
 678
 679         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 680         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 681 } /* ixv_enable_queue */
 682
 683 /************************************************************************
 684  * ixv_disable_queue
 685  ************************************************************************/
 686 static inline void
 687 ixv_disable_queue(struct adapter *adapter, u32 vector)
 688 {
 689         struct ixgbe_hw *hw = &adapter->hw;
 690         u64             queue = (u64)(1 << vector);
 691         u32             mask;
 692
 693         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 694         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
 695 } /* ixv_disable_queue */
 696
 697
 698 /************************************************************************
 699  * ixv_msix_que - MSI-X Queue Interrupt Service routine
 700  ************************************************************************/
 701 static int
 702 ixv_msix_que(void *arg)
 703 {
 704         struct ix_rx_queue *que = arg;
 705         struct adapter     *adapter = que->adapter;
 706
 707         ixv_disable_queue(adapter, que->msix);
 708         ++que->irqs;
 709
 710         return (FILTER_SCHEDULE_THREAD);
 711 } /* ixv_msix_que */
 712
 713 /************************************************************************
 714  * ixv_msix_mbx
 715  ************************************************************************/
 716 static int
 717 ixv_msix_mbx(void *arg)
 718 {
 719         struct adapter  *adapter = arg;
 720         struct ixgbe_hw *hw = &adapter->hw;
 721         u32             reg;
 722
 723         ++adapter->link_irq;
 724
 725         /* First get the cause */
 726         reg = IXGBE_READ_REG(hw, IXGBE_VTEICS);
 727         /* Clear interrupt with write */
 728         IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg);
 729
 730         /* Link status change */
 731         if (reg & IXGBE_EICR_LSC)
 732                 iflib_admin_intr_deferred(adapter->ctx);
 733
 734         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER);
 735
 736         return (FILTER_HANDLED);
 737 } /* ixv_msix_mbx */
 738
 739 /************************************************************************
 740  * ixv_media_status - Media Ioctl callback
 741  *
 742  *   Called whenever the user queries the status of
 743  *   the interface using ifconfig.
 744  ************************************************************************/
 745 static void
 746 ixv_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
 747 {
 748         struct adapter *adapter = iflib_get_softc(ctx);
 749
 750         INIT_DEBUGOUT("ixv_media_status: begin");
 751
 752         iflib_admin_intr_deferred(ctx);
 753
 754         ifmr->ifm_status = IFM_AVALID;
 755         ifmr->ifm_active = IFM_ETHER;
 756
 757         if (!adapter->link_active)
 758                 return;
 759
 760         ifmr->ifm_status |= IFM_ACTIVE;
 761
 762         switch (adapter->link_speed) {
 763                 case IXGBE_LINK_SPEED_1GB_FULL:
 764                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
 765                         break;
 766                 case IXGBE_LINK_SPEED_10GB_FULL:
 767                         ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
 768                         break;
 769                 case IXGBE_LINK_SPEED_100_FULL:
 770                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
 771                         break;
 772                 case IXGBE_LINK_SPEED_10_FULL:
 773                         ifmr->ifm_active |= IFM_10_T | IFM_FDX;
 774                         break;
 775         }
 776 } /* ixv_if_media_status */
 777
 778 /************************************************************************
 779  * ixv_if_media_change - Media Ioctl callback
 780  *
 781  *   Called when the user changes speed/duplex using
 782  *   media/mediopt option with ifconfig.
 783  ************************************************************************/
 784 static int
 785 ixv_if_media_change(if_ctx_t ctx)
 786 {
 787         struct adapter *adapter = iflib_get_softc(ctx);
 788         struct ifmedia *ifm = iflib_get_media(ctx);
 789
 790         INIT_DEBUGOUT("ixv_media_change: begin");
 791
 792         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 793                 return (EINVAL);
 794
 795         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 796         case IFM_AUTO:
 797                 break;
 798         default:
 799                 device_printf(adapter->dev, "Only auto media type\n");
 800                 return (EINVAL);
 801         }
 802
 803         return (0);
 804 } /* ixv_if_media_change */
 805
 806
 807 /************************************************************************
 808  * ixv_negotiate_api
 809  *
 810  *   Negotiate the Mailbox API with the PF;
 811  *   start with the most featured API first.
 812  ************************************************************************/
 813 static int
 814 ixv_negotiate_api(struct adapter *adapter)
 815 {
 816         struct ixgbe_hw *hw = &adapter->hw;
 817         int             mbx_api[] = { ixgbe_mbox_api_11,
 818                                       ixgbe_mbox_api_10,
 819                                       ixgbe_mbox_api_unknown };
 820         int             i = 0;
 821
 822         while (mbx_api[i] != ixgbe_mbox_api_unknown) {
 823                 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0)
 824                         return (0);
 825                 i++;
 826         }
 827
 828         return (EINVAL);
 829 } /* ixv_negotiate_api */
 830
 831
 832 /************************************************************************
 833  * ixv_if_multi_set - Multicast Update
 834  *
 835  *   Called whenever multicast address list is updated.
 836  ************************************************************************/
 837 static void
 838 ixv_if_multi_set(if_ctx_t ctx)
 839 {
 840         u8       mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
 841         struct adapter     *adapter = iflib_get_softc(ctx);
 842         u8                 *update_ptr;
 843         struct ifmultiaddr *ifma;
 844         if_t               ifp = iflib_get_ifp(ctx);
 845         int                mcnt = 0;
 846
 847         IOCTL_DEBUGOUT("ixv_if_multi_set: begin");
 848
 849         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 850                 if (ifma->ifma_addr->sa_family != AF_LINK)
 851                         continue;
 852                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 853                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
 854                     IXGBE_ETH_LENGTH_OF_ADDRESS);
 855                 mcnt++;
 856         }
 857
 858         update_ptr = mta;
 859
 860         adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, update_ptr, mcnt,
 861             ixv_mc_array_itr, TRUE);
 862 } /* ixv_if_multi_set */
 863
 864 /************************************************************************
 865  * ixv_mc_array_itr
 866  *
 867  *   An iterator function needed by the multicast shared code.
 868  *   It feeds the shared code routine the addresses in the
 869  *   array of ixv_set_multi() one by one.
 870  ************************************************************************/
 871 static u8 *
 872 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
 873 {
 874         u8 *addr = *update_ptr;
 875         u8 *newptr;
 876
 877         *vmdq = 0;
 878
 879         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
 880         *update_ptr = newptr;
 881
 882         return addr;
 883 } /* ixv_mc_array_itr */
 884
 885 /************************************************************************
 886  * ixv_if_local_timer - Timer routine
 887  *
 888  *   Checks for link status, updates statistics,
 889  *   and runs the watchdog check.
 890  ************************************************************************/
 891 static void
 892 ixv_if_local_timer(if_ctx_t ctx, uint16_t qid)
 893 {
 894         if (qid != 0)
 895                 return;
 896
 897         /* Fire off the adminq task */
 898         iflib_admin_intr_deferred(ctx);
 899 } /* ixv_if_local_timer */
 900
 901 /************************************************************************
 902  * ixv_if_update_admin_status - Update OS on link state
 903  *
 904  * Note: Only updates the OS on the cached link state.
 905  *       The real check of the hardware only happens with
 906  *       a link interrupt.
 907  ************************************************************************/
 908 static void
 909 ixv_if_update_admin_status(if_ctx_t ctx)
 910 {
 911         struct adapter *adapter = iflib_get_softc(ctx);
 912         device_t       dev = iflib_get_dev(ctx);
 913         s32            status;
 914
 915         adapter->hw.mac.get_link_status = TRUE;
 916
 917         status = ixgbe_check_link(&adapter->hw, &adapter->link_speed,
 918             &adapter->link_up, FALSE);
 919
 920         if (status != IXGBE_SUCCESS && adapter->hw.adapter_stopped == FALSE) {
 921                 /* Mailbox's Clear To Send status is lost or timeout occurred.
 922                  * We need reinitialization. */
 923                 iflib_get_ifp(ctx)->if_init(ctx);
 924         }
 925
 926         if (adapter->link_up) {
 927                 if (adapter->link_active == FALSE) {
 928                         if (bootverbose)
 929                                 device_printf(dev, "Link is up %d Gbps %s \n",
 930                                     ((adapter->link_speed == 128) ? 10 : 1),
 931                                     "Full Duplex");
 932                         adapter->link_active = TRUE;
 933                         iflib_link_state_change(ctx, LINK_STATE_UP,
 934                             IF_Gbps(10));
 935                 }
 936         } else { /* Link down */
 937                 if (adapter->link_active == TRUE) {
 938                         if (bootverbose)
 939                                 device_printf(dev, "Link is Down\n");
 940                         iflib_link_state_change(ctx, LINK_STATE_DOWN,  0);
 941                         adapter->link_active = FALSE;
 942                 }
 943         }
 944
 945         /* Stats Update */
 946         ixv_update_stats(adapter);
 947 } /* ixv_if_update_admin_status */
 948
 949
 950 /************************************************************************
 951  * ixv_if_stop - Stop the hardware
 952  *
 953  *   Disables all traffic on the adapter by issuing a
 954  *   global reset on the MAC and deallocates TX/RX buffers.
 955  ************************************************************************/
 956 static void
 957 ixv_if_stop(if_ctx_t ctx)
 958 {
 959         struct adapter  *adapter = iflib_get_softc(ctx);
 960         struct ixgbe_hw *hw = &adapter->hw;
 961
 962         INIT_DEBUGOUT("ixv_stop: begin\n");
 963
 964         ixv_if_disable_intr(ctx);
 965
 966         hw->mac.ops.reset_hw(hw);
 967         adapter->hw.adapter_stopped = FALSE;
 968         hw->mac.ops.stop_adapter(hw);
 969
 970         /* Update the stack */
 971         adapter->link_up = FALSE;
 972         ixv_if_update_admin_status(ctx);
 973
 974         /* reprogram the RAR[0] in case user changed it. */
 975         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
 976 } /* ixv_if_stop */
 977
 978
 979 /************************************************************************
 980  * ixv_identify_hardware - Determine hardware revision.
 981  ************************************************************************/
 982 static void
 983 ixv_identify_hardware(if_ctx_t ctx)
 984 {
 985         struct adapter  *adapter = iflib_get_softc(ctx);
 986         device_t        dev = iflib_get_dev(ctx);
 987         struct ixgbe_hw *hw = &adapter->hw;
 988
 989         /* Save off the information about this board */
 990         hw->vendor_id = pci_get_vendor(dev);
 991         hw->device_id = pci_get_device(dev);
 992         hw->revision_id = pci_get_revid(dev);
 993         hw->subsystem_vendor_id = pci_get_subvendor(dev);
 994         hw->subsystem_device_id = pci_get_subdevice(dev);
 995
 996         /* A subset of set_mac_type */
 997         switch (hw->device_id) {
 998         case IXGBE_DEV_ID_82599_VF:
 999                 hw->mac.type = ixgbe_mac_82599_vf;
1000                 break;
1001         case IXGBE_DEV_ID_X540_VF:
1002                 hw->mac.type = ixgbe_mac_X540_vf;
1003                 break;
1004         case IXGBE_DEV_ID_X550_VF:
1005                 hw->mac.type = ixgbe_mac_X550_vf;
1006                 break;
1007         case IXGBE_DEV_ID_X550EM_X_VF:
1008                 hw->mac.type = ixgbe_mac_X550EM_x_vf;
1009                 break;
1010         case IXGBE_DEV_ID_X550EM_A_VF:
1011                 hw->mac.type = ixgbe_mac_X550EM_a_vf;
1012                 break;
1013         default:
1014                 device_printf(dev, "unknown mac type\n");
1015                 hw->mac.type = ixgbe_mac_unknown;
1016                 break;
1017         }
1018 } /* ixv_identify_hardware */
1019
1020 /************************************************************************
1021  * ixv_if_msix_intr_assign - Setup MSI-X Interrupt resources and handlers
1022  ************************************************************************/
1023 static int
1024 ixv_if_msix_intr_assign(if_ctx_t ctx, int msix)
1025 {
1026         struct adapter     *adapter = iflib_get_softc(ctx);
1027         device_t           dev = iflib_get_dev(ctx);
1028         struct ix_rx_queue *rx_que = adapter->rx_queues;
1029         struct ix_tx_queue *tx_que;
1030         int                error, rid, vector = 0;
1031         char               buf[16];
1032
1033         for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rx_que++) {
1034                 rid = vector + 1;
1035
1036                 snprintf(buf, sizeof(buf), "rxq%d", i);
1037                 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
1038                     IFLIB_INTR_RX, ixv_msix_que, rx_que, rx_que->rxr.me, buf);
1039
1040                 if (error) {
1041                         device_printf(iflib_get_dev(ctx),
1042                             "Failed to allocate que int %d err: %d", i, error);
1043                         adapter->num_rx_queues = i + 1;
1044                         goto fail;
1045                 }
1046
1047                 rx_que->msix = vector;
1048                 adapter->active_queues |= (u64)(1 << rx_que->msix);
1049
1050         }
1051
1052         for (int i = 0; i < adapter->num_tx_queues; i++) {
1053                 snprintf(buf, sizeof(buf), "txq%d", i);
1054                 tx_que = &adapter->tx_queues[i];
1055                 tx_que->msix = i % adapter->num_rx_queues;
1056                 iflib_softirq_alloc_generic(ctx,
1057                     &adapter->rx_queues[tx_que->msix].que_irq,
1058                     IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
1059         }
1060         rid = vector + 1;
1061         error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid,
1062             IFLIB_INTR_ADMIN, ixv_msix_mbx, adapter, 0, "aq");
1063         if (error) {
1064                 device_printf(iflib_get_dev(ctx),
1065                     "Failed to register admin handler");
1066                 return (error);
1067         }
1068
1069         adapter->vector = vector;
1070         /*
1071          * Due to a broken design QEMU will fail to properly
1072          * enable the guest for MSIX unless the vectors in
1073          * the table are all set up, so we must rewrite the
1074          * ENABLE in the MSIX control register again at this
1075          * point to cause it to successfully initialize us.
1076          */
1077         if (adapter->hw.mac.type == ixgbe_mac_82599_vf) {
1078                 int msix_ctrl;
1079                 pci_find_cap(dev, PCIY_MSIX, &rid);
1080                 rid += PCIR_MSIX_CTRL;
1081                 msix_ctrl = pci_read_config(dev, rid, 2);
1082                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1083                 pci_write_config(dev, rid, msix_ctrl, 2);
1084         }
1085
1086         return (0);
1087
1088 fail:
1089         iflib_irq_free(ctx, &adapter->irq);
1090         rx_que = adapter->rx_queues;
1091         for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++)
1092                 iflib_irq_free(ctx, &rx_que->que_irq);
1093
1094         return (error);
1095 } /* ixv_if_msix_intr_assign */
1096
1097 /************************************************************************
1098  * ixv_allocate_pci_resources
1099  ************************************************************************/
1100 static int
1101 ixv_allocate_pci_resources(if_ctx_t ctx)
1102 {
1103         struct adapter *adapter = iflib_get_softc(ctx);
1104         device_t       dev = iflib_get_dev(ctx);
1105         int            rid;
1106
1107         rid = PCIR_BAR(0);
1108         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1109             RF_ACTIVE);
1110
1111         if (!(adapter->pci_mem)) {
1112                 device_printf(dev, "Unable to allocate bus resource: memory\n");
1113                 return (ENXIO);
1114         }
1115
1116         adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->pci_mem);
1117         adapter->osdep.mem_bus_space_handle =
1118             rman_get_bushandle(adapter->pci_mem);
1119         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
1120
1121         return (0);
1122 } /* ixv_allocate_pci_resources */
1123
1124 /************************************************************************
1125  * ixv_free_pci_resources
1126  ************************************************************************/
1127 static void
1128 ixv_free_pci_resources(if_ctx_t ctx)
1129 {
1130         struct adapter     *adapter = iflib_get_softc(ctx);
1131         struct ix_rx_queue *que = adapter->rx_queues;
1132         device_t           dev = iflib_get_dev(ctx);
1133
1134         /* Release all msix queue resources */
1135         if (adapter->intr_type == IFLIB_INTR_MSIX)
1136                 iflib_irq_free(ctx, &adapter->irq);
1137
1138         if (que != NULL) {
1139                 for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
1140                         iflib_irq_free(ctx, &que->que_irq);
1141                 }
1142         }
1143
1144         /* Clean the Legacy or Link interrupt last */
1145         if (adapter->pci_mem != NULL)
1146                 bus_release_resource(dev, SYS_RES_MEMORY,
1147                                      PCIR_BAR(0), adapter->pci_mem);
1148 } /* ixv_free_pci_resources */
1149
1150 /************************************************************************
1151  * ixv_setup_interface
1152  *
1153  *   Setup networking device structure and register an interface.
1154  ************************************************************************/
1155 static int
1156 ixv_setup_interface(if_ctx_t ctx)
1157 {
1158         struct adapter *adapter = iflib_get_softc(ctx);
1159         if_softc_ctx_t scctx = adapter->shared;
1160         struct ifnet   *ifp = iflib_get_ifp(ctx);
1161
1162         INIT_DEBUGOUT("ixv_setup_interface: begin");
1163
1164         if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
1165         if_setbaudrate(ifp, IF_Gbps(10));
1166         ifp->if_snd.ifq_maxlen = scctx->isc_ntxd[0] - 2;
1167
1168
1169         adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR;
1170         ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1171         ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
1172
1173         return 0;
1174 } /* ixv_setup_interface */
1175
1176 /************************************************************************
1177  * ixv_if_get_counter
1178  ************************************************************************/
1179 static uint64_t
1180 ixv_if_get_counter(if_ctx_t ctx, ift_counter cnt)
1181 {
1182         struct adapter *adapter = iflib_get_softc(ctx);
1183         if_t           ifp = iflib_get_ifp(ctx);
1184
1185         switch (cnt) {
1186         case IFCOUNTER_IPACKETS:
1187                 return (adapter->ipackets);
1188         case IFCOUNTER_OPACKETS:
1189                 return (adapter->opackets);
1190         case IFCOUNTER_IBYTES:
1191                 return (adapter->ibytes);
1192         case IFCOUNTER_OBYTES:
1193                 return (adapter->obytes);
1194         case IFCOUNTER_IMCASTS:
1195                 return (adapter->imcasts);
1196         default:
1197                 return (if_get_counter_default(ifp, cnt));
1198         }
1199 } /* ixv_if_get_counter */
1200
1201 /************************************************************************
1202  * ixv_initialize_transmit_units - Enable transmit unit.
1203  ************************************************************************/
1204 static void
1205 ixv_initialize_transmit_units(if_ctx_t ctx)
1206 {
1207         struct adapter     *adapter = iflib_get_softc(ctx);
1208         struct ixgbe_hw    *hw = &adapter->hw;
1209         if_softc_ctx_t     scctx = adapter->shared;
1210         struct ix_tx_queue *que = adapter->tx_queues;
1211         int                i;
1212
1213         for (i = 0; i < adapter->num_tx_queues; i++, que++) {
1214                 struct tx_ring *txr = &que->txr;
1215                 u64            tdba = txr->tx_paddr;
1216                 u32            txctrl, txdctl;
1217                 int            j = txr->me;
1218
1219                 /* Set WTHRESH to 8, burst writeback */
1220                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1221                 txdctl |= (8 << 16);
1222                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1223
1224                 /* Set the HW Tx Head and Tail indices */
1225                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0);
1226                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0);
1227
1228                 /* Set Tx Tail register */
1229                 txr->tail = IXGBE_VFTDT(j);
1230
1231                 txr->tx_rs_cidx = txr->tx_rs_pidx = txr->tx_cidx_processed = 0;
1232                 for (int k = 0; k < scctx->isc_ntxd[0]; k++)
1233                         txr->tx_rsq[k] = QIDX_INVALID;
1234
1235                 /* Set Ring parameters */
1236                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1237                     (tdba & 0x00000000ffffffffULL));
1238                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1239                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j),
1240                     scctx->isc_ntxd[0] * sizeof(struct ixgbe_legacy_tx_desc));
1241                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1242                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
1243                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1244
1245                 /* Now enable */
1246                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1247                 txdctl |= IXGBE_TXDCTL_ENABLE;
1248                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1249         }
1250
1251         return;
1252 } /* ixv_initialize_transmit_units */
1253
1254 /************************************************************************
1255  * ixv_initialize_rss_mapping
1256  ************************************************************************/
1257 static void
1258 ixv_initialize_rss_mapping(struct adapter *adapter)
1259 {
1260         struct ixgbe_hw *hw = &adapter->hw;
1261         u32             reta = 0, mrqc, rss_key[10];
1262         int             queue_id;
1263         int             i, j;
1264         u32             rss_hash_config;
1265
1266         if (adapter->feat_en & IXGBE_FEATURE_RSS) {
1267                 /* Fetch the configured RSS key */
1268                 rss_getkey((uint8_t *)&rss_key);
1269         } else {
1270                 /* set up random bits */
1271                 arc4rand(&rss_key, sizeof(rss_key), 0);
1272         }
1273
1274         /* Now fill out hash function seeds */
1275         for (i = 0; i < 10; i++)
1276                 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]);
1277
1278         /* Set up the redirection table */
1279         for (i = 0, j = 0; i < 64; i++, j++) {
1280                 if (j == adapter->num_rx_queues)
1281                         j = 0;
1282
1283                 if (adapter->feat_en & IXGBE_FEATURE_RSS) {
1284                         /*
1285                          * Fetch the RSS bucket id for the given indirection
1286                          * entry. Cap it at the number of configured buckets
1287                          * (which is num_rx_queues.)
1288                          */
1289                         queue_id = rss_get_indirection_to_bucket(i);
1290                         queue_id = queue_id % adapter->num_rx_queues;
1291                 } else
1292                         queue_id = j;
1293
1294                 /*
1295                  * The low 8 bits are for hash value (n+0);
1296                  * The next 8 bits are for hash value (n+1), etc.
1297                  */
1298                 reta >>= 8;
1299                 reta |= ((uint32_t)queue_id) << 24;
1300                 if ((i & 3) == 3) {
1301                         IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta);
1302                         reta = 0;
1303                 }
1304         }
1305
1306         /* Perform hash on these packet types */
1307         if (adapter->feat_en & IXGBE_FEATURE_RSS)
1308                 rss_hash_config = rss_gethashconfig();
1309         else {
1310                 /*
1311                  * Disable UDP - IP fragments aren't currently being handled
1312                  * and so we end up with a mix of 2-tuple and 4-tuple
1313                  * traffic.
1314                  */
1315                 rss_hash_config = RSS_HASHTYPE_RSS_IPV4
1316                                 | RSS_HASHTYPE_RSS_TCP_IPV4
1317                                 | RSS_HASHTYPE_RSS_IPV6
1318                                 | RSS_HASHTYPE_RSS_TCP_IPV6;
1319         }
1320
1321         mrqc = IXGBE_MRQC_RSSEN;
1322         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
1323                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
1324         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
1325                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
1326         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
1327                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
1328         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
1329                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
1330         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
1331                 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n",
1332                     __func__);
1333         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
1334                 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n",
1335                     __func__);
1336         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
1337                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
1338         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
1339                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
1340         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
1341                 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n",
1342                     __func__);
1343         IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc);
1344 } /* ixv_initialize_rss_mapping */
1345
1346
1347 /************************************************************************
1348  * ixv_initialize_receive_units - Setup receive registers and features.
1349  ************************************************************************/
1350 static void
1351 ixv_initialize_receive_units(if_ctx_t ctx)
1352 {
1353         struct adapter     *adapter = iflib_get_softc(ctx);
1354         if_softc_ctx_t     scctx;
1355         struct ixgbe_hw    *hw = &adapter->hw;
1356         struct ifnet       *ifp = iflib_get_ifp(ctx);
1357         struct ix_rx_queue *que = adapter->rx_queues;
1358         u32                bufsz, psrtype;
1359
1360         if (ifp->if_mtu > ETHERMTU)
1361                 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1362         else
1363                 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1364
1365         psrtype = IXGBE_PSRTYPE_TCPHDR
1366                 | IXGBE_PSRTYPE_UDPHDR
1367                 | IXGBE_PSRTYPE_IPV4HDR
1368                 | IXGBE_PSRTYPE_IPV6HDR
1369                 | IXGBE_PSRTYPE_L2HDR;
1370
1371         if (adapter->num_rx_queues > 1)
1372                 psrtype |= 1 << 29;
1373
1374         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1375
1376         /* Tell PF our max_frame size */
1377         if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) {
1378                 device_printf(adapter->dev, "There is a problem with the PF setup.  It is likely the receive unit for this VF will not function correctly.\n");
1379         }
1380         scctx = adapter->shared;
1381
1382         for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
1383                 struct rx_ring *rxr = &que->rxr;
1384                 u64            rdba = rxr->rx_paddr;
1385                 u32            reg, rxdctl;
1386                 int            j = rxr->me;
1387
1388                 /* Disable the queue */
1389                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1390                 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1391                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1392                 for (int k = 0; k < 10; k++) {
1393                         if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
1394                             IXGBE_RXDCTL_ENABLE)
1395                                 msec_delay(1);
1396                         else
1397                                 break;
1398                 }
1399                 wmb();
1400                 /* Setup the Base and Length of the Rx Descriptor Ring */
1401                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1402                     (rdba & 0x00000000ffffffffULL));
1403                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1404                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j),
1405                     scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));
1406
1407                 /* Reset the ring indices */
1408                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
1409                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0);
1410
1411                 /* Set up the SRRCTL register */
1412                 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j));
1413                 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
1414                 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
1415                 reg |= bufsz;
1416                 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1417                 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg);
1418
1419                 /* Capture Rx Tail index */
1420                 rxr->tail = IXGBE_VFRDT(rxr->me);
1421
1422                 /* Do the queue enabling last */
1423                 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1424                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1425                 for (int l = 0; l < 10; l++) {
1426                         if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
1427                             IXGBE_RXDCTL_ENABLE)
1428                                 break;
1429                         msec_delay(1);
1430                 }
1431                 wmb();
1432
1433                 /* Set the Tail Pointer */
1434 #ifdef DEV_NETMAP
1435                 /*
1436                  * In netmap mode, we must preserve the buffers made
1437                  * available to userspace before the if_init()
1438                  * (this is true by default on the TX side, because
1439                  * init makes all buffers available to userspace).
1440                  *
1441                  * netmap_reset() and the device specific routines
1442                  * (e.g. ixgbe_setup_receive_rings()) map these
1443                  * buffers at the end of the NIC ring, so here we
1444                  * must set the RDT (tail) register to make sure
1445                  * they are not overwritten.
1446                  *
1447                  * In this driver the NIC ring starts at RDH = 0,
1448                  * RDT points to the last slot available for reception (?),
1449                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1450                  */
1451                 if (ifp->if_capenable & IFCAP_NETMAP) {
1452                         struct netmap_adapter *na = NA(ifp);
1453                         struct netmap_kring *kring = na->rx_rings[j];
1454                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1455
1456                         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t);
1457                 } else
1458 #endif /* DEV_NETMAP */
1459                         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
1460                             scctx->isc_nrxd[0] - 1);
1461         }
1462
1463         ixv_initialize_rss_mapping(adapter);
1464 } /* ixv_initialize_receive_units */
1465
1466 /************************************************************************
1467  * ixv_setup_vlan_support
1468  ************************************************************************/
1469 static void
1470 ixv_setup_vlan_support(if_ctx_t ctx)
1471 {
1472         struct adapter  *adapter = iflib_get_softc(ctx);
1473         struct ixgbe_hw *hw = &adapter->hw;
1474         u32             ctrl, vid, vfta, retry;
1475
1476         /*
1477          * We get here thru if_init, meaning
1478          * a soft reset, this has already cleared
1479          * the VFTA and other state, so if there
1480          * have been no vlan's registered do nothing.
1481          */
1482         if (adapter->num_vlans == 0)
1483                 return;
1484
1485         /* Enable the queues */
1486         for (int i = 0; i < adapter->num_rx_queues; i++) {
1487                 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
1488                 ctrl |= IXGBE_RXDCTL_VME;
1489                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl);
1490                 /*
1491                  * Let Rx path know that it needs to store VLAN tag
1492                  * as part of extra mbuf info.
1493                  */
1494                 adapter->rx_queues[i].rxr.vtag_strip = TRUE;
1495         }
1496
1497         /*
1498          * A soft reset zero's out the VFTA, so
1499          * we need to repopulate it now.
1500          */
1501         for (int i = 0; i < IXGBE_VFTA_SIZE; i++) {
1502                 if (ixv_shadow_vfta[i] == 0)
1503                         continue;
1504                 vfta = ixv_shadow_vfta[i];
1505                 /*
1506                  * Reconstruct the vlan id's
1507                  * based on the bits set in each
1508                  * of the array ints.
1509                  */
1510                 for (int j = 0; j < 32; j++) {
1511                         retry = 0;
1512                         if ((vfta & (1 << j)) == 0)
1513                                 continue;
1514                         vid = (i * 32) + j;
1515                         /* Call the shared code mailbox routine */
1516                         while (hw->mac.ops.set_vfta(hw, vid, 0, TRUE, FALSE)) {
1517                                 if (++retry > 5)
1518                                         break;
1519                         }
1520                 }
1521         }
1522 } /* ixv_setup_vlan_support */
1523
1524 /************************************************************************
1525  * ixv_if_register_vlan
1526  *
1527  *   Run via a vlan config EVENT, it enables us to use the
1528  *   HW Filter table since we can get the vlan id. This just
1529  *   creates the entry in the soft version of the VFTA, init
1530  *   will repopulate the real table.
1531  ************************************************************************/
1532 static void
1533 ixv_if_register_vlan(if_ctx_t ctx, u16 vtag)
1534 {
1535         struct adapter *adapter = iflib_get_softc(ctx);
1536         u16            index, bit;
1537
1538         index = (vtag >> 5) & 0x7F;
1539         bit = vtag & 0x1F;
1540         ixv_shadow_vfta[index] |= (1 << bit);
1541         ++adapter->num_vlans;
1542 } /* ixv_if_register_vlan */
1543
1544 /************************************************************************
1545  * ixv_if_unregister_vlan
1546  *
1547  *   Run via a vlan unconfig EVENT, remove our entry
1548  *   in the soft vfta.
1549  ************************************************************************/
1550 static void
1551 ixv_if_unregister_vlan(if_ctx_t ctx, u16 vtag)
1552 {
1553         struct adapter *adapter = iflib_get_softc(ctx);
1554         u16            index, bit;
1555
1556         index = (vtag >> 5) & 0x7F;
1557         bit = vtag & 0x1F;
1558         ixv_shadow_vfta[index] &= ~(1 << bit);
1559         --adapter->num_vlans;
1560 } /* ixv_if_unregister_vlan */
1561
1562 /************************************************************************
1563  * ixv_if_enable_intr
1564  ************************************************************************/
1565 static void
1566 ixv_if_enable_intr(if_ctx_t ctx)
1567 {
1568         struct adapter  *adapter = iflib_get_softc(ctx);
1569         struct ixgbe_hw *hw = &adapter->hw;
1570         struct ix_rx_queue *que = adapter->rx_queues;
1571         u32             mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1572
1573         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
1574
1575         mask = IXGBE_EIMS_ENABLE_MASK;
1576         mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1577         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
1578
1579         for (int i = 0; i < adapter->num_rx_queues; i++, que++)
1580                 ixv_enable_queue(adapter, que->msix);
1581
1582         IXGBE_WRITE_FLUSH(hw);
1583 } /* ixv_if_enable_intr */
1584
1585 /************************************************************************
1586  * ixv_if_disable_intr
1587  ************************************************************************/
1588 static void
1589 ixv_if_disable_intr(if_ctx_t ctx)
1590 {
1591         struct adapter *adapter = iflib_get_softc(ctx);
1592         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0);
1593         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0);
1594         IXGBE_WRITE_FLUSH(&adapter->hw);
1595 } /* ixv_if_disable_intr */
1596
1597 /************************************************************************
1598  * ixv_if_rx_queue_intr_enable
1599  ************************************************************************/
1600 static int
1601 ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
1602 {
1603         struct adapter  *adapter = iflib_get_softc(ctx);
1604         struct ix_rx_queue *que = &adapter->rx_queues[rxqid];
1605
1606         ixv_enable_queue(adapter, que->rxr.me);
1607
1608         return (0);
1609 } /* ixv_if_rx_queue_intr_enable */
1610
1611 /************************************************************************
1612  * ixv_set_ivar
1613  *
1614  *   Setup the correct IVAR register for a particular MSI-X interrupt
1615  *    - entry is the register array entry
1616  *    - vector is the MSI-X vector for this queue
1617  *    - type is RX/TX/MISC
1618  ************************************************************************/
1619 static void
1620 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
1621 {
1622         struct ixgbe_hw *hw = &adapter->hw;
1623         u32             ivar, index;
1624
1625         vector |= IXGBE_IVAR_ALLOC_VAL;
1626
1627         if (type == -1) { /* MISC IVAR */
1628                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
1629                 ivar &= ~0xFF;
1630                 ivar |= vector;
1631                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
1632         } else {          /* RX/TX IVARS */
1633                 index = (16 * (entry & 1)) + (8 * type);
1634                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
1635                 ivar &= ~(0xFF << index);
1636                 ivar |= (vector << index);
1637                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
1638         }
1639 } /* ixv_set_ivar */
1640
1641 /************************************************************************
1642  * ixv_configure_ivars
1643  ************************************************************************/
1644 static void
1645 ixv_configure_ivars(struct adapter *adapter)
1646 {
1647         struct ix_rx_queue *que = adapter->rx_queues;
1648
1649         MPASS(adapter->num_rx_queues == adapter->num_tx_queues);
1650
1651         for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
1652                 /* First the RX queue entry */
1653                 ixv_set_ivar(adapter, i, que->msix, 0);
1654                 /* ... and the TX */
1655                 ixv_set_ivar(adapter, i, que->msix, 1);
1656                 /* Set an initial value in EITR */
1657                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(que->msix),
1658                     IXGBE_EITR_DEFAULT);
1659         }
1660
1661         /* For the mailbox interrupt */
1662         ixv_set_ivar(adapter, 1, adapter->vector, -1);
1663 } /* ixv_configure_ivars */
1664
1665 /************************************************************************
1666  * ixv_save_stats
1667  *
1668  *   The VF stats registers never have a truly virgin
1669  *   starting point, so this routine tries to make an
1670  *   artificial one, marking ground zero on attach as
1671  *   it were.
1672  ************************************************************************/
1673 static void
1674 ixv_save_stats(struct adapter *adapter)
1675 {
1676         if (adapter->stats.vf.vfgprc || adapter->stats.vf.vfgptc) {
1677                 adapter->stats.vf.saved_reset_vfgprc +=
1678                     adapter->stats.vf.vfgprc - adapter->stats.vf.base_vfgprc;
1679                 adapter->stats.vf.saved_reset_vfgptc +=
1680                     adapter->stats.vf.vfgptc - adapter->stats.vf.base_vfgptc;
1681                 adapter->stats.vf.saved_reset_vfgorc +=
1682                     adapter->stats.vf.vfgorc - adapter->stats.vf.base_vfgorc;
1683                 adapter->stats.vf.saved_reset_vfgotc +=
1684                     adapter->stats.vf.vfgotc - adapter->stats.vf.base_vfgotc;
1685                 adapter->stats.vf.saved_reset_vfmprc +=
1686                     adapter->stats.vf.vfmprc - adapter->stats.vf.base_vfmprc;
1687         }
1688 } /* ixv_save_stats */
1689
1690 /************************************************************************
1691  * ixv_init_stats
1692  ************************************************************************/
1693 static void
1694 ixv_init_stats(struct adapter *adapter)
1695 {
1696         struct ixgbe_hw *hw = &adapter->hw;
1697
1698         adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1699         adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1700         adapter->stats.vf.last_vfgorc |=
1701             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1702
1703         adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1704         adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1705         adapter->stats.vf.last_vfgotc |=
1706             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1707
1708         adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1709
1710         adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc;
1711         adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc;
1712         adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc;
1713         adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc;
1714         adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc;
1715 } /* ixv_init_stats */
1716
1717 #define UPDATE_STAT_32(reg, last, count)                \
1718 {                                                       \
1719         u32 current = IXGBE_READ_REG(hw, reg);          \
1720         if (current < last)                             \
1721                 count += 0x100000000LL;                 \
1722         last = current;                                 \
1723         count &= 0xFFFFFFFF00000000LL;                  \
1724         count |= current;                               \
1725 }
1726
1727 #define UPDATE_STAT_36(lsb, msb, last, count)           \
1728 {                                                       \
1729         u64 cur_lsb = IXGBE_READ_REG(hw, lsb);          \
1730         u64 cur_msb = IXGBE_READ_REG(hw, msb);          \
1731         u64 current = ((cur_msb << 32) | cur_lsb);      \
1732         if (current < last)                             \
1733                 count += 0x1000000000LL;                \
1734         last = current;                                 \
1735         count &= 0xFFFFFFF000000000LL;                  \
1736         count |= current;                               \
1737 }
1738
1739 /************************************************************************
1740  * ixv_update_stats - Update the board statistics counters.
1741  ************************************************************************/
1742 void
1743 ixv_update_stats(struct adapter *adapter)
1744 {
1745         struct ixgbe_hw *hw = &adapter->hw;
1746         struct ixgbevf_hw_stats *stats = &adapter->stats.vf;
1747
1748         UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.vf.last_vfgprc,
1749             adapter->stats.vf.vfgprc);
1750         UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.vf.last_vfgptc,
1751             adapter->stats.vf.vfgptc);
1752         UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1753             adapter->stats.vf.last_vfgorc, adapter->stats.vf.vfgorc);
1754         UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1755             adapter->stats.vf.last_vfgotc, adapter->stats.vf.vfgotc);
1756         UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.vf.last_vfmprc,
1757             adapter->stats.vf.vfmprc);
1758
1759         /* Fill out the OS statistics structure */
1760         IXGBE_SET_IPACKETS(adapter, stats->vfgprc);
1761         IXGBE_SET_OPACKETS(adapter, stats->vfgptc);
1762         IXGBE_SET_IBYTES(adapter, stats->vfgorc);
1763         IXGBE_SET_OBYTES(adapter, stats->vfgotc);
1764         IXGBE_SET_IMCASTS(adapter, stats->vfmprc);
1765 } /* ixv_update_stats */
1766
1767 /************************************************************************
1768  * ixv_add_stats_sysctls - Add statistic sysctls for the VF.
1769  ************************************************************************/
1770 static void
1771 ixv_add_stats_sysctls(struct adapter *adapter)
1772 {
1773         device_t                dev = adapter->dev;
1774         struct ix_tx_queue      *tx_que = adapter->tx_queues;
1775         struct ix_rx_queue      *rx_que = adapter->rx_queues;
1776         struct sysctl_ctx_list  *ctx = device_get_sysctl_ctx(dev);
1777         struct sysctl_oid       *tree = device_get_sysctl_tree(dev);
1778         struct sysctl_oid_list  *child = SYSCTL_CHILDREN(tree);
1779         struct ixgbevf_hw_stats *stats = &adapter->stats.vf;
1780         struct sysctl_oid       *stat_node, *queue_node;
1781         struct sysctl_oid_list  *stat_list, *queue_list;
1782
1783 #define QUEUE_NAME_LEN 32
1784         char                    namebuf[QUEUE_NAME_LEN];
1785
1786         /* Driver Statistics */
1787         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
1788             CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts");
1789         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
1790             CTLFLAG_RD, &adapter->link_irq, "Link MSI-X IRQ Handled");
1791
1792         for (int i = 0; i < adapter->num_tx_queues; i++, tx_que++) {
1793                 struct tx_ring *txr = &tx_que->txr;
1794                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1795                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1796                     CTLFLAG_RD, NULL, "Queue Name");
1797                 queue_list = SYSCTL_CHILDREN(queue_node);
1798
1799                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
1800                     CTLFLAG_RD, &(txr->tso_tx), "TSO Packets");
1801                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
1802                     CTLFLAG_RD, &(txr->total_packets), "TX Packets");
1803         }
1804
1805         for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++) {
1806                 struct rx_ring *rxr = &rx_que->rxr;
1807                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1808                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1809                     CTLFLAG_RD, NULL, "Queue Name");
1810                 queue_list = SYSCTL_CHILDREN(queue_node);
1811
1812                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
1813                     CTLFLAG_RD, &(rx_que->irqs), "IRQs on queue");
1814                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
1815                     CTLFLAG_RD, &(rxr->rx_packets), "RX packets");
1816                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
1817                     CTLFLAG_RD, &(rxr->rx_bytes), "RX bytes");
1818                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
1819                     CTLFLAG_RD, &(rxr->rx_discarded), "Discarded RX packets");
1820         }
1821
1822         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
1823             CTLFLAG_RD, NULL, "VF Statistics (read from HW registers)");
1824         stat_list = SYSCTL_CHILDREN(stat_node);
1825
1826         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
1827             CTLFLAG_RD, &stats->vfgprc, "Good Packets Received");
1828         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
1829             CTLFLAG_RD, &stats->vfgorc, "Good Octets Received");
1830         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
1831             CTLFLAG_RD, &stats->vfmprc, "Multicast Packets Received");
1832         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
1833             CTLFLAG_RD, &stats->vfgptc, "Good Packets Transmitted");
1834         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
1835             CTLFLAG_RD, &stats->vfgotc, "Good Octets Transmitted");
1836 } /* ixv_add_stats_sysctls */
1837
1838 /************************************************************************
1839  * ixv_print_debug_info
1840  *
1841  *   Called only when em_display_debug_stats is enabled.
1842  *   Provides a way to take a look at important statistics
1843  *   maintained by the driver and hardware.
1844  ************************************************************************/
1845 static void
1846 ixv_print_debug_info(struct adapter *adapter)
1847 {
1848         device_t        dev = adapter->dev;
1849         struct ixgbe_hw *hw = &adapter->hw;
1850
1851         device_printf(dev, "Error Byte Count = %u \n",
1852             IXGBE_READ_REG(hw, IXGBE_ERRBC));
1853
1854         device_printf(dev, "MBX IRQ Handled: %lu\n", (long)adapter->link_irq);
1855 } /* ixv_print_debug_info */
1856
1857 /************************************************************************
1858  * ixv_sysctl_debug
1859  ************************************************************************/
1860 static int
1861 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)
1862 {
1863         struct adapter *adapter;
1864         int            error, result;
1865
1866         result = -1;
1867         error = sysctl_handle_int(oidp, &result, 0, req);
1868
1869         if (error || !req->newptr)
1870                 return (error);
1871
1872         if (result == 1) {
1873                 adapter = (struct adapter *)arg1;
1874                 ixv_print_debug_info(adapter);
1875         }
1876
1877         return error;
1878 } /* ixv_sysctl_debug */
1879
1880 /************************************************************************
1881  * ixv_init_device_features
1882  ************************************************************************/
1883 static void
1884 ixv_init_device_features(struct adapter *adapter)
1885 {
1886         adapter->feat_cap = IXGBE_FEATURE_NETMAP
1887                           | IXGBE_FEATURE_VF
1888                           | IXGBE_FEATURE_RSS
1889                           | IXGBE_FEATURE_LEGACY_TX;
1890
1891         /* A tad short on feature flags for VFs, atm. */
1892         switch (adapter->hw.mac.type) {
1893         case ixgbe_mac_82599_vf:
1894                 break;
1895         case ixgbe_mac_X540_vf:
1896                 break;
1897         case ixgbe_mac_X550_vf:
1898         case ixgbe_mac_X550EM_x_vf:
1899         case ixgbe_mac_X550EM_a_vf:
1900                 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD;
1901                 break;
1902         default:
1903                 break;
1904         }
1905
1906         /* Enabled by default... */
1907         /* Is a virtual function (VF) */
1908         if (adapter->feat_cap & IXGBE_FEATURE_VF)
1909                 adapter->feat_en |= IXGBE_FEATURE_VF;
1910         /* Netmap */
1911         if (adapter->feat_cap & IXGBE_FEATURE_NETMAP)
1912                 adapter->feat_en |= IXGBE_FEATURE_NETMAP;
1913         /* Receive-Side Scaling (RSS) */
1914         if (adapter->feat_cap & IXGBE_FEATURE_RSS)
1915                 adapter->feat_en |= IXGBE_FEATURE_RSS;
1916         /* Needs advanced context descriptor regardless of offloads req'd */
1917         if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD)
1918                 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD;
1919 } /* ixv_init_device_features */
1920