sys/dev/ixgbe/if_ixv.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #ifndef IXGBE_STANDALONE_BUILD
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #endif
  40
  41 #include "ixgbe.h"
  42
  43 /*********************************************************************
  44  *  Driver version
  45  *********************************************************************/
  46 char ixv_driver_version[] = "1.4.0";
  47
  48 /*********************************************************************
  49  *  PCI Device ID Table
  50  *
  51  *  Used by probe to select devices to load on
  52  *  Last field stores an index into ixv_strings
  53  *  Last entry must be all 0s
  54  *
  55  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  56  *********************************************************************/
  57
  58 static ixgbe_vendor_info_t ixv_vendor_info_array[] =
  59 {
  60         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0},
  61         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0},
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0},
  64         /* required last entry */
  65         {0, 0, 0, 0, 0}
  66 };
  67
  68 /*********************************************************************
  69  *  Table of branding strings
  70  *********************************************************************/
  71
  72 static char    *ixv_strings[] = {
  73         "Intel(R) PRO/10GbE Virtual Function Network Driver"
  74 };
  75
  76 /*********************************************************************
  77  *  Function prototypes
  78  *********************************************************************/
  79 static int      ixv_probe(device_t);
  80 static int      ixv_attach(device_t);
  81 static int      ixv_detach(device_t);
  82 static int      ixv_shutdown(device_t);
  83 static int      ixv_ioctl(struct ifnet *, u_long, caddr_t);
  84 static void     ixv_init(void *);
  85 static void     ixv_init_locked(struct adapter *);
  86 static void     ixv_stop(void *);
  87 static void     ixv_media_status(struct ifnet *, struct ifmediareq *);
  88 static int      ixv_media_change(struct ifnet *);
  89 static void     ixv_identify_hardware(struct adapter *);
  90 static int      ixv_allocate_pci_resources(struct adapter *);
  91 static int      ixv_allocate_msix(struct adapter *);
  92 static int      ixv_setup_msix(struct adapter *);
  93 static void     ixv_free_pci_resources(struct adapter *);
  94 static void     ixv_local_timer(void *);
  95 static void     ixv_setup_interface(device_t, struct adapter *);
  96 static void     ixv_config_link(struct adapter *);
  97
  98 static void     ixv_initialize_transmit_units(struct adapter *);
  99 static void     ixv_initialize_receive_units(struct adapter *);
 100
 101 static void     ixv_enable_intr(struct adapter *);
 102 static void     ixv_disable_intr(struct adapter *);
 103 static void     ixv_set_multi(struct adapter *);
 104 static void     ixv_update_link_status(struct adapter *);
 105 static int      ixv_sysctl_debug(SYSCTL_HANDLER_ARGS);
 106 static void     ixv_set_ivar(struct adapter *, u8, u8, s8);
 107 static void     ixv_configure_ivars(struct adapter *);
 108 static u8 *     ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 109
 110 static void     ixv_setup_vlan_support(struct adapter *);
 111 static void     ixv_register_vlan(void *, struct ifnet *, u16);
 112 static void     ixv_unregister_vlan(void *, struct ifnet *, u16);
 113
 114 static void     ixv_save_stats(struct adapter *);
 115 static void     ixv_init_stats(struct adapter *);
 116 static void     ixv_update_stats(struct adapter *);
 117 static void     ixv_add_stats_sysctls(struct adapter *);
 118
 119 /* The MSI/X Interrupt handlers */
 120 static void     ixv_msix_que(void *);
 121 static void     ixv_msix_mbx(void *);
 122
 123 /* Deferred interrupt tasklets */
 124 static void     ixv_handle_que(void *, int);
 125 static void     ixv_handle_mbx(void *, int);
 126
 127 /*********************************************************************
 128  *  FreeBSD Device Interface Entry Points
 129  *********************************************************************/
 130
 131 static device_method_t ixv_methods[] = {
 132         /* Device interface */
 133         DEVMETHOD(device_probe, ixv_probe),
 134         DEVMETHOD(device_attach, ixv_attach),
 135         DEVMETHOD(device_detach, ixv_detach),
 136         DEVMETHOD(device_shutdown, ixv_shutdown),
 137         DEVMETHOD_END
 138 };
 139
 140 static driver_t ixv_driver = {
 141         "ixv", ixv_methods, sizeof(struct adapter),
 142 };
 143
 144 devclass_t ixv_devclass;
 145 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0);
 146 MODULE_DEPEND(ixv, pci, 1, 1, 1);
 147 MODULE_DEPEND(ixv, ether, 1, 1, 1);
 148 /* XXX depend on 'ix' ? */
 149
 150 /*
 151 ** TUNEABLE PARAMETERS:
 152 */
 153
 154 /* Number of Queues - do not exceed MSIX vectors - 1 */
 155 static int ixv_num_queues = 1;
 156 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues);
 157
 158 /*
 159 ** AIM: Adaptive Interrupt Moderation
 160 ** which means that the interrupt rate
 161 ** is varied over time based on the
 162 ** traffic for that interrupt vector
 163 */
 164 static int ixv_enable_aim = FALSE;
 165 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim);
 166
 167 /* How many packets rxeof tries to clean at a time */
 168 static int ixv_rx_process_limit = 256;
 169 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit);
 170
 171 /* How many packets txeof tries to clean at a time */
 172 static int ixv_tx_process_limit = 256;
 173 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit);
 174
 175 /* Flow control setting, default to full */
 176 static int ixv_flow_control = ixgbe_fc_full;
 177 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control);
 178
 179 /*
 180  * Header split: this causes the hardware to DMA
 181  * the header into a seperate mbuf from the payload,
 182  * it can be a performance win in some workloads, but
 183  * in others it actually hurts, its off by default.
 184  */
 185 static int ixv_header_split = FALSE;
 186 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split);
 187
 188 /*
 189 ** Number of TX descriptors per ring,
 190 ** setting higher than RX as this seems
 191 ** the better performing choice.
 192 */
 193 static int ixv_txd = DEFAULT_TXD;
 194 TUNABLE_INT("hw.ixv.txd", &ixv_txd);
 195
 196 /* Number of RX descriptors per ring */
 197 static int ixv_rxd = DEFAULT_RXD;
 198 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd);
 199
 200 /*
 201 ** Shadow VFTA table, this is needed because
 202 ** the real filter table gets cleared during
 203 ** a soft reset and we need to repopulate it.
 204 */
 205 static u32 ixv_shadow_vfta[IXGBE_VFTA_SIZE];
 206
 207 /*********************************************************************
 208  *  Device identification routine
 209  *
 210  *  ixv_probe determines if the driver should be loaded on
 211  *  adapter based on PCI vendor/device id of the adapter.
 212  *
 213  *  return BUS_PROBE_DEFAULT on success, positive on failure
 214  *********************************************************************/
 215
 216 static int
 217 ixv_probe(device_t dev)
 218 {
 219         ixgbe_vendor_info_t *ent;
 220
 221         u16     pci_vendor_id = 0;
 222         u16     pci_device_id = 0;
 223         u16     pci_subvendor_id = 0;
 224         u16     pci_subdevice_id = 0;
 225         char    adapter_name[256];
 226
 227
 228         pci_vendor_id = pci_get_vendor(dev);
 229         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 230                 return (ENXIO);
 231
 232         pci_device_id = pci_get_device(dev);
 233         pci_subvendor_id = pci_get_subvendor(dev);
 234         pci_subdevice_id = pci_get_subdevice(dev);
 235
 236         ent = ixv_vendor_info_array;
 237         while (ent->vendor_id != 0) {
 238                 if ((pci_vendor_id == ent->vendor_id) &&
 239                     (pci_device_id == ent->device_id) &&
 240
 241                     ((pci_subvendor_id == ent->subvendor_id) ||
 242                      (ent->subvendor_id == 0)) &&
 243
 244                     ((pci_subdevice_id == ent->subdevice_id) ||
 245                      (ent->subdevice_id == 0))) {
 246                         sprintf(adapter_name, "%s, Version - %s",
 247                                 ixv_strings[ent->index],
 248                                 ixv_driver_version);
 249                         device_set_desc_copy(dev, adapter_name);
 250                         return (BUS_PROBE_DEFAULT);
 251                 }
 252                 ent++;
 253         }
 254         return (ENXIO);
 255 }
 256
 257 /*********************************************************************
 258  *  Device initialization routine
 259  *
 260  *  The attach entry point is called when the driver is being loaded.
 261  *  This routine identifies the type of hardware, allocates all resources
 262  *  and initializes the hardware.
 263  *
 264  *  return 0 on success, positive on failure
 265  *********************************************************************/
 266
 267 static int
 268 ixv_attach(device_t dev)
 269 {
 270         struct adapter *adapter;
 271         struct ixgbe_hw *hw;
 272         int             error = 0;
 273
 274         INIT_DEBUGOUT("ixv_attach: begin");
 275
 276         /* Allocate, clear, and link in our adapter structure */
 277         adapter = device_get_softc(dev);
 278         adapter->dev = adapter->osdep.dev = dev;
 279         hw = &adapter->hw;
 280
 281         /* Core Lock Init*/
 282         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 283
 284         /* SYSCTL APIs */
 285         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 286                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 287                         OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW,
 288                         adapter, 0, ixv_sysctl_debug, "I", "Debug Info");
 289
 290         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 291                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 292                         OID_AUTO, "enable_aim", CTLFLAG_RW,
 293                         &ixv_enable_aim, 1, "Interrupt Moderation");
 294
 295         /* Set up the timer callout */
 296         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 297
 298         /* Determine hardware revision */
 299         ixv_identify_hardware(adapter);
 300
 301         /* Do base PCI setup - map BAR0 */
 302         if (ixv_allocate_pci_resources(adapter)) {
 303                 device_printf(dev, "Allocation of PCI resources failed\n");
 304                 error = ENXIO;
 305                 goto err_out;
 306         }
 307
 308         /* Do descriptor calc and sanity checks */
 309         if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 310             ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) {
 311                 device_printf(dev, "TXD config issue, using default!\n");
 312                 adapter->num_tx_desc = DEFAULT_TXD;
 313         } else
 314                 adapter->num_tx_desc = ixv_txd;
 315
 316         if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 317             ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) {
 318                 device_printf(dev, "RXD config issue, using default!\n");
 319                 adapter->num_rx_desc = DEFAULT_RXD;
 320         } else
 321                 adapter->num_rx_desc = ixv_rxd;
 322
 323         /* Allocate our TX/RX Queues */
 324         if (ixgbe_allocate_queues(adapter)) {
 325                 error = ENOMEM;
 326                 goto err_out;
 327         }
 328
 329         /*
 330         ** Initialize the shared code: its
 331         ** at this point the mac type is set.
 332         */
 333         error = ixgbe_init_shared_code(hw);
 334         if (error) {
 335                 device_printf(dev,"Shared Code Initialization Failure\n");
 336                 error = EIO;
 337                 goto err_late;
 338         }
 339
 340         /* Setup the mailbox */
 341         ixgbe_init_mbx_params_vf(hw);
 342
 343         ixgbe_reset_hw(hw);
 344
 345         /* Get the Mailbox API version */
 346         device_printf(dev,"MBX API %d negotiation: %d\n",
 347             ixgbe_mbox_api_11,
 348             ixgbevf_negotiate_api_version(hw, ixgbe_mbox_api_11));
 349
 350         error = ixgbe_init_hw(hw);
 351         if (error) {
 352                 device_printf(dev,"Hardware Initialization Failure\n");
 353                 error = EIO;
 354                 goto err_late;
 355         }
 356
 357         error = ixv_allocate_msix(adapter);
 358         if (error)
 359                 goto err_late;
 360
 361         /* If no mac address was assigned, make a random one */
 362         if (!ixv_check_ether_addr(hw->mac.addr)) {
 363                 u8 addr[ETHER_ADDR_LEN];
 364                 arc4rand(&addr, sizeof(addr), 0);
 365                 addr[0] &= 0xFE;
 366                 addr[0] |= 0x02;
 367                 bcopy(addr, hw->mac.addr, sizeof(addr));
 368         }
 369
 370         /* Setup OS specific network interface */
 371         ixv_setup_interface(dev, adapter);
 372
 373         /* Do the stats setup */
 374         ixv_save_stats(adapter);
 375         ixv_init_stats(adapter);
 376         ixv_add_stats_sysctls(adapter);
 377
 378         /* Register for VLAN events */
 379         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 380             ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 381         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 382             ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 383
 384         INIT_DEBUGOUT("ixv_attach: end");
 385         return (0);
 386
 387 err_late:
 388         ixgbe_free_transmit_structures(adapter);
 389         ixgbe_free_receive_structures(adapter);
 390 err_out:
 391         ixv_free_pci_resources(adapter);
 392         return (error);
 393
 394 }
 395
 396 /*********************************************************************
 397  *  Device removal routine
 398  *
 399  *  The detach entry point is called when the driver is being removed.
 400  *  This routine stops the adapter and deallocates all the resources
 401  *  that were allocated for driver operation.
 402  *
 403  *  return 0 on success, positive on failure
 404  *********************************************************************/
 405
 406 static int
 407 ixv_detach(device_t dev)
 408 {
 409         struct adapter *adapter = device_get_softc(dev);
 410         struct ix_queue *que = adapter->queues;
 411
 412         INIT_DEBUGOUT("ixv_detach: begin");
 413
 414         /* Make sure VLANS are not using driver */
 415         if (adapter->ifp->if_vlantrunk != NULL) {
 416                 device_printf(dev,"Vlan in use, detach first\n");
 417                 return (EBUSY);
 418         }
 419
 420         IXGBE_CORE_LOCK(adapter);
 421         ixv_stop(adapter);
 422         IXGBE_CORE_UNLOCK(adapter);
 423
 424         for (int i = 0; i < adapter->num_queues; i++, que++) {
 425                 if (que->tq) {
 426                         struct tx_ring  *txr = que->txr;
 427                         taskqueue_drain(que->tq, &txr->txq_task);
 428                         taskqueue_drain(que->tq, &que->que_task);
 429                         taskqueue_free(que->tq);
 430                 }
 431         }
 432
 433         /* Drain the Mailbox(link) queue */
 434         if (adapter->tq) {
 435                 taskqueue_drain(adapter->tq, &adapter->link_task);
 436                 taskqueue_free(adapter->tq);
 437         }
 438
 439         /* Unregister VLAN events */
 440         if (adapter->vlan_attach != NULL)
 441                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 442         if (adapter->vlan_detach != NULL)
 443                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 444
 445         ether_ifdetach(adapter->ifp);
 446         callout_drain(&adapter->timer);
 447         ixv_free_pci_resources(adapter);
 448         bus_generic_detach(dev);
 449         if_free(adapter->ifp);
 450
 451         ixgbe_free_transmit_structures(adapter);
 452         ixgbe_free_receive_structures(adapter);
 453
 454         IXGBE_CORE_LOCK_DESTROY(adapter);
 455         return (0);
 456 }
 457
 458 /*********************************************************************
 459  *
 460  *  Shutdown entry point
 461  *
 462  **********************************************************************/
 463 static int
 464 ixv_shutdown(device_t dev)
 465 {
 466         struct adapter *adapter = device_get_softc(dev);
 467         IXGBE_CORE_LOCK(adapter);
 468         ixv_stop(adapter);
 469         IXGBE_CORE_UNLOCK(adapter);
 470         return (0);
 471 }
 472
 473
 474 /*********************************************************************
 475  *  Ioctl entry point
 476  *
 477  *  ixv_ioctl is called when the user wants to configure the
 478  *  interface.
 479  *
 480  *  return 0 on success, positive on failure
 481  **********************************************************************/
 482
 483 static int
 484 ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 485 {
 486         struct adapter  *adapter = ifp->if_softc;
 487         struct ifreq    *ifr = (struct ifreq *) data;
 488 #if defined(INET) || defined(INET6)
 489         struct ifaddr   *ifa = (struct ifaddr *) data;
 490         bool            avoid_reset = FALSE;
 491 #endif
 492         int             error = 0;
 493
 494         switch (command) {
 495
 496         case SIOCSIFADDR:
 497 #ifdef INET
 498                 if (ifa->ifa_addr->sa_family == AF_INET)
 499                         avoid_reset = TRUE;
 500 #endif
 501 #ifdef INET6
 502                 if (ifa->ifa_addr->sa_family == AF_INET6)
 503                         avoid_reset = TRUE;
 504 #endif
 505 #if defined(INET) || defined(INET6)
 506                 /*
 507                 ** Calling init results in link renegotiation,
 508                 ** so we avoid doing it when possible.
 509                 */
 510                 if (avoid_reset) {
 511                         ifp->if_flags |= IFF_UP;
 512                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 513                                 ixv_init(adapter);
 514                         if (!(ifp->if_flags & IFF_NOARP))
 515                                 arp_ifinit(ifp, ifa);
 516                 } else
 517                         error = ether_ioctl(ifp, command, data);
 518                 break;
 519 #endif
 520         case SIOCSIFMTU:
 521                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 522                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 523                         error = EINVAL;
 524                 } else {
 525                         IXGBE_CORE_LOCK(adapter);
 526                         ifp->if_mtu = ifr->ifr_mtu;
 527                         adapter->max_frame_size =
 528                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 529                         ixv_init_locked(adapter);
 530                         IXGBE_CORE_UNLOCK(adapter);
 531                 }
 532                 break;
 533         case SIOCSIFFLAGS:
 534                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 535                 IXGBE_CORE_LOCK(adapter);
 536                 if (ifp->if_flags & IFF_UP) {
 537                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 538                                 ixv_init_locked(adapter);
 539                 } else
 540                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 541                                 ixv_stop(adapter);
 542                 adapter->if_flags = ifp->if_flags;
 543                 IXGBE_CORE_UNLOCK(adapter);
 544                 break;
 545         case SIOCADDMULTI:
 546         case SIOCDELMULTI:
 547                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 548                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 549                         IXGBE_CORE_LOCK(adapter);
 550                         ixv_disable_intr(adapter);
 551                         ixv_set_multi(adapter);
 552                         ixv_enable_intr(adapter);
 553                         IXGBE_CORE_UNLOCK(adapter);
 554                 }
 555                 break;
 556         case SIOCSIFMEDIA:
 557         case SIOCGIFMEDIA:
 558                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 559                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 560                 break;
 561         case SIOCSIFCAP:
 562         {
 563                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 564                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 565                 if (mask & IFCAP_HWCSUM)
 566                         ifp->if_capenable ^= IFCAP_HWCSUM;
 567                 if (mask & IFCAP_TSO4)
 568                         ifp->if_capenable ^= IFCAP_TSO4;
 569                 if (mask & IFCAP_LRO)
 570                         ifp->if_capenable ^= IFCAP_LRO;
 571                 if (mask & IFCAP_VLAN_HWTAGGING)
 572                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 573                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 574                         IXGBE_CORE_LOCK(adapter);
 575                         ixv_init_locked(adapter);
 576                         IXGBE_CORE_UNLOCK(adapter);
 577                 }
 578                 VLAN_CAPABILITIES(ifp);
 579                 break;
 580         }
 581
 582         default:
 583                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 584                 error = ether_ioctl(ifp, command, data);
 585                 break;
 586         }
 587
 588         return (error);
 589 }
 590
 591 /*********************************************************************
 592  *  Init entry point
 593  *
 594  *  This routine is used in two ways. It is used by the stack as
 595  *  init entry point in network interface structure. It is also used
 596  *  by the driver as a hw/sw initialization routine to get to a
 597  *  consistent state.
 598  *
 599  *  return 0 on success, positive on failure
 600  **********************************************************************/
 601 #define IXGBE_MHADD_MFS_SHIFT 16
 602
 603 static void
 604 ixv_init_locked(struct adapter *adapter)
 605 {
 606         struct ifnet    *ifp = adapter->ifp;
 607         device_t        dev = adapter->dev;
 608         struct ixgbe_hw *hw = &adapter->hw;
 609         u32             mhadd, gpie;
 610
 611         INIT_DEBUGOUT("ixv_init: begin");
 612         mtx_assert(&adapter->core_mtx, MA_OWNED);
 613         hw->adapter_stopped = FALSE;
 614         ixgbe_stop_adapter(hw);
 615         callout_stop(&adapter->timer);
 616
 617         /* reprogram the RAR[0] in case user changed it. */
 618         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
 619
 620         /* Get the latest mac address, User can use a LAA */
 621         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
 622              IXGBE_ETH_LENGTH_OF_ADDRESS);
 623         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
 624         hw->addr_ctrl.rar_used_count = 1;
 625
 626         /* Prepare transmit descriptors and buffers */
 627         if (ixgbe_setup_transmit_structures(adapter)) {
 628                 device_printf(dev,"Could not setup transmit structures\n");
 629                 ixv_stop(adapter);
 630                 return;
 631         }
 632
 633         ixgbe_reset_hw(hw);
 634         ixv_initialize_transmit_units(adapter);
 635
 636         /* Setup Multicast table */
 637         ixv_set_multi(adapter);
 638
 639         /*
 640         ** Determine the correct mbuf pool
 641         ** for doing jumbo/headersplit
 642         */
 643         if (ifp->if_mtu > ETHERMTU)
 644                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 645         else
 646                 adapter->rx_mbuf_sz = MCLBYTES;
 647
 648         /* Prepare receive descriptors and buffers */
 649         if (ixgbe_setup_receive_structures(adapter)) {
 650                 device_printf(dev,"Could not setup receive structures\n");
 651                 ixv_stop(adapter);
 652                 return;
 653         }
 654
 655         /* Configure RX settings */
 656         ixv_initialize_receive_units(adapter);
 657
 658         /* Enable Enhanced MSIX mode */
 659         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
 660         gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME;
 661         gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD;
 662         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
 663
 664         /* Set the various hardware offload abilities */
 665         ifp->if_hwassist = 0;
 666         if (ifp->if_capenable & IFCAP_TSO4)
 667                 ifp->if_hwassist |= CSUM_TSO;
 668         if (ifp->if_capenable & IFCAP_TXCSUM) {
 669                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 670 #if __FreeBSD_version >= 800000
 671                 ifp->if_hwassist |= CSUM_SCTP;
 672 #endif
 673         }
 674
 675         /* Set MTU size */
 676         if (ifp->if_mtu > ETHERMTU) {
 677                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
 678                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
 679                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
 680                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
 681         }
 682
 683         /* Set up VLAN offload and filter */
 684         ixv_setup_vlan_support(adapter);
 685
 686         callout_reset(&adapter->timer, hz, ixv_local_timer, adapter);
 687
 688         /* Set up MSI/X routing */
 689         ixv_configure_ivars(adapter);
 690
 691         /* Set up auto-mask */
 692         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE);
 693
 694         /* Set moderation on the Link interrupt */
 695         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->vector), IXGBE_LINK_ITR);
 696
 697         /* Stats init */
 698         ixv_init_stats(adapter);
 699
 700         /* Config/Enable Link */
 701         ixv_config_link(adapter);
 702
 703         /* And now turn on interrupts */
 704         ixv_enable_intr(adapter);
 705
 706         /* Now inform the stack we're ready */
 707         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 708         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 709
 710         return;
 711 }
 712
 713 static void
 714 ixv_init(void *arg)
 715 {
 716         struct adapter *adapter = arg;
 717
 718         IXGBE_CORE_LOCK(adapter);
 719         ixv_init_locked(adapter);
 720         IXGBE_CORE_UNLOCK(adapter);
 721         return;
 722 }
 723
 724
 725 /*
 726 **
 727 ** MSIX Interrupt Handlers and Tasklets
 728 **
 729 */
 730
 731 static inline void
 732 ixv_enable_queue(struct adapter *adapter, u32 vector)
 733 {
 734         struct ixgbe_hw *hw = &adapter->hw;
 735         u32     queue = 1 << vector;
 736         u32     mask;
 737
 738         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 739         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 740 }
 741
 742 static inline void
 743 ixv_disable_queue(struct adapter *adapter, u32 vector)
 744 {
 745         struct ixgbe_hw *hw = &adapter->hw;
 746         u64     queue = (u64)(1 << vector);
 747         u32     mask;
 748
 749         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 750         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
 751 }
 752
 753 static inline void
 754 ixv_rearm_queues(struct adapter *adapter, u64 queues)
 755 {
 756         u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
 757         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask);
 758 }
 759
 760
 761 static void
 762 ixv_handle_que(void *context, int pending)
 763 {
 764         struct ix_queue *que = context;
 765         struct adapter  *adapter = que->adapter;
 766         struct tx_ring  *txr = que->txr;
 767         struct ifnet    *ifp = adapter->ifp;
 768         bool            more;
 769
 770         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 771                 more = ixgbe_rxeof(que);
 772                 IXGBE_TX_LOCK(txr);
 773                 ixgbe_txeof(txr);
 774 #if __FreeBSD_version >= 800000
 775                 if (!drbr_empty(ifp, txr->br))
 776                         ixgbe_mq_start_locked(ifp, txr);
 777 #else
 778                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 779                         ixgbe_start_locked(txr, ifp);
 780 #endif
 781                 IXGBE_TX_UNLOCK(txr);
 782                 if (more) {
 783                         taskqueue_enqueue(que->tq, &que->que_task);
 784                         return;
 785                 }
 786         }
 787
 788         /* Reenable this interrupt */
 789         ixv_enable_queue(adapter, que->msix);
 790         return;
 791 }
 792
 793 /*********************************************************************
 794  *
 795  *  MSI Queue Interrupt Service routine
 796  *
 797  **********************************************************************/
 798 void
 799 ixv_msix_que(void *arg)
 800 {
 801         struct ix_queue *que = arg;
 802         struct adapter  *adapter = que->adapter;
 803         struct ifnet    *ifp = adapter->ifp;
 804         struct tx_ring  *txr = que->txr;
 805         struct rx_ring  *rxr = que->rxr;
 806         bool            more;
 807         u32             newitr = 0;
 808
 809         ixv_disable_queue(adapter, que->msix);
 810         ++que->irqs;
 811
 812         more = ixgbe_rxeof(que);
 813
 814         IXGBE_TX_LOCK(txr);
 815         ixgbe_txeof(txr);
 816         /*
 817         ** Make certain that if the stack
 818         ** has anything queued the task gets
 819         ** scheduled to handle it.
 820         */
 821 #ifdef IXGBE_LEGACY_TX
 822         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
 823                 ixgbe_start_locked(txr, ifp);
 824 #else
 825         if (!drbr_empty(adapter->ifp, txr->br))
 826                 ixgbe_mq_start_locked(ifp, txr);
 827 #endif
 828         IXGBE_TX_UNLOCK(txr);
 829
 830         /* Do AIM now? */
 831
 832         if (ixv_enable_aim == FALSE)
 833                 goto no_calc;
 834         /*
 835         ** Do Adaptive Interrupt Moderation:
 836         **  - Write out last calculated setting
 837         **  - Calculate based on average size over
 838         **    the last interval.
 839         */
 840         if (que->eitr_setting)
 841                 IXGBE_WRITE_REG(&adapter->hw,
 842                     IXGBE_VTEITR(que->msix),
 843                     que->eitr_setting);
 844
 845         que->eitr_setting = 0;
 846
 847         /* Idle, do nothing */
 848         if ((txr->bytes == 0) && (rxr->bytes == 0))
 849                 goto no_calc;
 850
 851         if ((txr->bytes) && (txr->packets))
 852                 newitr = txr->bytes/txr->packets;
 853         if ((rxr->bytes) && (rxr->packets))
 854                 newitr = max(newitr,
 855                     (rxr->bytes / rxr->packets));
 856         newitr += 24; /* account for hardware frame, crc */
 857
 858         /* set an upper boundary */
 859         newitr = min(newitr, 3000);
 860
 861         /* Be nice to the mid range */
 862         if ((newitr > 300) && (newitr < 1200))
 863                 newitr = (newitr / 3);
 864         else
 865                 newitr = (newitr / 2);
 866
 867         newitr |= newitr << 16;
 868
 869         /* save for next interrupt */
 870         que->eitr_setting = newitr;
 871
 872         /* Reset state */
 873         txr->bytes = 0;
 874         txr->packets = 0;
 875         rxr->bytes = 0;
 876         rxr->packets = 0;
 877
 878 no_calc:
 879         if (more)
 880                 taskqueue_enqueue(que->tq, &que->que_task);
 881         else /* Reenable this interrupt */
 882                 ixv_enable_queue(adapter, que->msix);
 883         return;
 884 }
 885
 886 static void
 887 ixv_msix_mbx(void *arg)
 888 {
 889         struct adapter  *adapter = arg;
 890         struct ixgbe_hw *hw = &adapter->hw;
 891         u32             reg;
 892
 893         ++adapter->link_irq;
 894
 895         /* First get the cause */
 896         reg = IXGBE_READ_REG(hw, IXGBE_VTEICS);
 897         /* Clear interrupt with write */
 898         IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg);
 899
 900         /* Link status change */
 901         if (reg & IXGBE_EICR_LSC)
 902                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
 903
 904         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER);
 905         return;
 906 }
 907
 908 /*********************************************************************
 909  *
 910  *  Media Ioctl callback
 911  *
 912  *  This routine is called whenever the user queries the status of
 913  *  the interface using ifconfig.
 914  *
 915  **********************************************************************/
 916 static void
 917 ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
 918 {
 919         struct adapter *adapter = ifp->if_softc;
 920
 921         INIT_DEBUGOUT("ixv_media_status: begin");
 922         IXGBE_CORE_LOCK(adapter);
 923         ixv_update_link_status(adapter);
 924
 925         ifmr->ifm_status = IFM_AVALID;
 926         ifmr->ifm_active = IFM_ETHER;
 927
 928         if (!adapter->link_active) {
 929                 IXGBE_CORE_UNLOCK(adapter);
 930                 return;
 931         }
 932
 933         ifmr->ifm_status |= IFM_ACTIVE;
 934
 935         switch (adapter->link_speed) {
 936                 case IXGBE_LINK_SPEED_1GB_FULL:
 937                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
 938                         break;
 939                 case IXGBE_LINK_SPEED_10GB_FULL:
 940                         ifmr->ifm_active |= IFM_FDX;
 941                         break;
 942         }
 943
 944         IXGBE_CORE_UNLOCK(adapter);
 945
 946         return;
 947 }
 948
 949 /*********************************************************************
 950  *
 951  *  Media Ioctl callback
 952  *
 953  *  This routine is called when the user changes speed/duplex using
 954  *  media/mediopt option with ifconfig.
 955  *
 956  **********************************************************************/
 957 static int
 958 ixv_media_change(struct ifnet * ifp)
 959 {
 960         struct adapter *adapter = ifp->if_softc;
 961         struct ifmedia *ifm = &adapter->media;
 962
 963         INIT_DEBUGOUT("ixv_media_change: begin");
 964
 965         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 966                 return (EINVAL);
 967
 968         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 969         case IFM_AUTO:
 970                 break;
 971         default:
 972                 device_printf(adapter->dev, "Only auto media type\n");
 973                 return (EINVAL);
 974         }
 975
 976         return (0);
 977 }
 978
 979
 980 /*********************************************************************
 981  *  Multicast Update
 982  *
 983  *  This routine is called whenever multicast address list is updated.
 984  *
 985  **********************************************************************/
 986 #define IXGBE_RAR_ENTRIES 16
 987
 988 static void
 989 ixv_set_multi(struct adapter *adapter)
 990 {
 991         u8      mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
 992         u8      *update_ptr;
 993         struct  ifmultiaddr *ifma;
 994         int     mcnt = 0;
 995         struct ifnet   *ifp = adapter->ifp;
 996
 997         IOCTL_DEBUGOUT("ixv_set_multi: begin");
 998
 999 #if __FreeBSD_version < 800000
1000         IF_ADDR_LOCK(ifp);
1001 #else
1002         if_maddr_rlock(ifp);
1003 #endif
1004         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1005                 if (ifma->ifma_addr->sa_family != AF_LINK)
1006                         continue;
1007                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1008                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1009                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1010                 mcnt++;
1011         }
1012 #if __FreeBSD_version < 800000
1013         IF_ADDR_UNLOCK(ifp);
1014 #else
1015         if_maddr_runlock(ifp);
1016 #endif
1017
1018         update_ptr = mta;
1019
1020         ixgbe_update_mc_addr_list(&adapter->hw,
1021             update_ptr, mcnt, ixv_mc_array_itr, TRUE);
1022
1023         return;
1024 }
1025
1026 /*
1027  * This is an iterator function now needed by the multicast
1028  * shared code. It simply feeds the shared code routine the
1029  * addresses in the array of ixv_set_multi() one by one.
1030  */
1031 static u8 *
1032 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1033 {
1034         u8 *addr = *update_ptr;
1035         u8 *newptr;
1036         *vmdq = 0;
1037
1038         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1039         *update_ptr = newptr;
1040         return addr;
1041 }
1042
1043 /*********************************************************************
1044  *  Timer routine
1045  *
1046  *  This routine checks for link status,updates statistics,
1047  *  and runs the watchdog check.
1048  *
1049  **********************************************************************/
1050
1051 static void
1052 ixv_local_timer(void *arg)
1053 {
1054         struct adapter  *adapter = arg;
1055         device_t        dev = adapter->dev;
1056         struct ix_queue *que = adapter->queues;
1057         u64             queues = 0;
1058         int             hung = 0;
1059
1060         mtx_assert(&adapter->core_mtx, MA_OWNED);
1061
1062         ixv_update_link_status(adapter);
1063
1064         /* Stats Update */
1065         ixv_update_stats(adapter);
1066
1067         /*
1068         ** Check the TX queues status
1069         **      - mark hung queues so we don't schedule on them
1070         **      - watchdog only if all queues show hung
1071         */
1072         for (int i = 0; i < adapter->num_queues; i++, que++) {
1073                 /* Keep track of queues with work for soft irq */
1074                 if (que->txr->busy)
1075                         queues |= ((u64)1 << que->me);
1076                 /*
1077                 ** Each time txeof runs without cleaning, but there
1078                 ** are uncleaned descriptors it increments busy. If
1079                 ** we get to the MAX we declare it hung.
1080                 */
1081                 if (que->busy == IXGBE_QUEUE_HUNG) {
1082                         ++hung;
1083                         /* Mark the queue as inactive */
1084                         adapter->active_queues &= ~((u64)1 << que->me);
1085                         continue;
1086                 } else {
1087                         /* Check if we've come back from hung */
1088                         if ((adapter->active_queues & ((u64)1 << que->me)) == 0)
1089                                 adapter->active_queues |= ((u64)1 << que->me);
1090                 }
1091                 if (que->busy >= IXGBE_MAX_TX_BUSY) {
1092                         device_printf(dev,"Warning queue %d "
1093                             "appears to be hung!\n", i);
1094                         que->txr->busy = IXGBE_QUEUE_HUNG;
1095                         ++hung;
1096                 }
1097
1098         }
1099
1100         /* Only truely watchdog if all queues show hung */
1101         if (hung == adapter->num_queues)
1102                 goto watchdog;
1103         else if (queues != 0) { /* Force an IRQ on queues with work */
1104                 ixv_rearm_queues(adapter, queues);
1105         }
1106
1107         callout_reset(&adapter->timer, hz, ixv_local_timer, adapter);
1108         return;
1109
1110 watchdog:
1111         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1112         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1113         adapter->watchdog_events++;
1114         ixv_init_locked(adapter);
1115 }
1116
1117 /*
1118 ** Note: this routine updates the OS on the link state
1119 **      the real check of the hardware only happens with
1120 **      a link interrupt.
1121 */
1122 static void
1123 ixv_update_link_status(struct adapter *adapter)
1124 {
1125         struct ifnet    *ifp = adapter->ifp;
1126         device_t dev = adapter->dev;
1127
1128         if (adapter->link_up){
1129                 if (adapter->link_active == FALSE) {
1130                         if (bootverbose)
1131                                 device_printf(dev,"Link is up %d Gbps %s \n",
1132                                     ((adapter->link_speed == 128)? 10:1),
1133                                     "Full Duplex");
1134                         adapter->link_active = TRUE;
1135                         if_link_state_change(ifp, LINK_STATE_UP);
1136                 }
1137         } else { /* Link down */
1138                 if (adapter->link_active == TRUE) {
1139                         if (bootverbose)
1140                                 device_printf(dev,"Link is Down\n");
1141                         if_link_state_change(ifp, LINK_STATE_DOWN);
1142                         adapter->link_active = FALSE;
1143                 }
1144         }
1145
1146         return;
1147 }
1148
1149
1150 /*********************************************************************
1151  *
1152  *  This routine disables all traffic on the adapter by issuing a
1153  *  global reset on the MAC and deallocates TX/RX buffers.
1154  *
1155  **********************************************************************/
1156
1157 static void
1158 ixv_stop(void *arg)
1159 {
1160         struct ifnet   *ifp;
1161         struct adapter *adapter = arg;
1162         struct ixgbe_hw *hw = &adapter->hw;
1163         ifp = adapter->ifp;
1164
1165         mtx_assert(&adapter->core_mtx, MA_OWNED);
1166
1167         INIT_DEBUGOUT("ixv_stop: begin\n");
1168         ixv_disable_intr(adapter);
1169
1170         /* Tell the stack that the interface is no longer active */
1171         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1172
1173         ixgbe_reset_hw(hw);
1174         adapter->hw.adapter_stopped = FALSE;
1175         ixgbe_stop_adapter(hw);
1176         callout_stop(&adapter->timer);
1177
1178         /* reprogram the RAR[0] in case user changed it. */
1179         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1180
1181         return;
1182 }
1183
1184
1185 /*********************************************************************
1186  *
1187  *  Determine hardware revision.
1188  *
1189  **********************************************************************/
1190 static void
1191 ixv_identify_hardware(struct adapter *adapter)
1192 {
1193         device_t        dev = adapter->dev;
1194         struct ixgbe_hw *hw = &adapter->hw;
1195
1196         /*
1197         ** Make sure BUSMASTER is set, on a VM under
1198         ** KVM it may not be and will break things.
1199         */
1200         pci_enable_busmaster(dev);
1201
1202         /* Save off the information about this board */
1203         hw->vendor_id = pci_get_vendor(dev);
1204         hw->device_id = pci_get_device(dev);
1205         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
1206         hw->subsystem_vendor_id =
1207             pci_read_config(dev, PCIR_SUBVEND_0, 2);
1208         hw->subsystem_device_id =
1209             pci_read_config(dev, PCIR_SUBDEV_0, 2);
1210
1211         /* We need this to determine device-specific things */
1212         ixgbe_set_mac_type(hw);
1213
1214         /* Set the right number of segments */
1215         adapter->num_segs = IXGBE_82599_SCATTER;
1216
1217         return;
1218 }
1219
1220 /*********************************************************************
1221  *
1222  *  Setup MSIX Interrupt resources and handlers
1223  *
1224  **********************************************************************/
1225 static int
1226 ixv_allocate_msix(struct adapter *adapter)
1227 {
1228         device_t        dev = adapter->dev;
1229         struct          ix_queue *que = adapter->queues;
1230         struct          tx_ring *txr = adapter->tx_rings;
1231         int             error, rid, vector = 0;
1232
1233         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
1234                 rid = vector + 1;
1235                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1236                     RF_SHAREABLE | RF_ACTIVE);
1237                 if (que->res == NULL) {
1238                         device_printf(dev,"Unable to allocate"
1239                             " bus resource: que interrupt [%d]\n", vector);
1240                         return (ENXIO);
1241                 }
1242                 /* Set the handler function */
1243                 error = bus_setup_intr(dev, que->res,
1244                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
1245                     ixv_msix_que, que, &que->tag);
1246                 if (error) {
1247                         que->res = NULL;
1248                         device_printf(dev, "Failed to register QUE handler");
1249                         return (error);
1250                 }
1251 #if __FreeBSD_version >= 800504
1252                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
1253 #endif
1254                 que->msix = vector;
1255                 adapter->active_queues |= (u64)(1 << que->msix);
1256                 /*
1257                 ** Bind the msix vector, and thus the
1258                 ** ring to the corresponding cpu.
1259                 */
1260                 if (adapter->num_queues > 1)
1261                         bus_bind_intr(dev, que->res, i);
1262                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
1263                 TASK_INIT(&que->que_task, 0, ixv_handle_que, que);
1264                 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT,
1265                     taskqueue_thread_enqueue, &que->tq);
1266                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1267                     device_get_nameunit(adapter->dev));
1268         }
1269
1270         /* and Mailbox */
1271         rid = vector + 1;
1272         adapter->res = bus_alloc_resource_any(dev,
1273             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
1274         if (!adapter->res) {
1275                 device_printf(dev,"Unable to allocate"
1276             " bus resource: MBX interrupt [%d]\n", rid);
1277                 return (ENXIO);
1278         }
1279         /* Set the mbx handler function */
1280         error = bus_setup_intr(dev, adapter->res,
1281             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1282             ixv_msix_mbx, adapter, &adapter->tag);
1283         if (error) {
1284                 adapter->res = NULL;
1285                 device_printf(dev, "Failed to register LINK handler");
1286                 return (error);
1287         }
1288 #if __FreeBSD_version >= 800504
1289         bus_describe_intr(dev, adapter->res, adapter->tag, "mbx");
1290 #endif
1291         adapter->vector = vector;
1292         /* Tasklets for Mailbox */
1293         TASK_INIT(&adapter->link_task, 0, ixv_handle_mbx, adapter);
1294         adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT,
1295             taskqueue_thread_enqueue, &adapter->tq);
1296         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq",
1297             device_get_nameunit(adapter->dev));
1298         /*
1299         ** Due to a broken design QEMU will fail to properly
1300         ** enable the guest for MSIX unless the vectors in
1301         ** the table are all set up, so we must rewrite the
1302         ** ENABLE in the MSIX control register again at this
1303         ** point to cause it to successfully initialize us.
1304         */
1305         if (adapter->hw.mac.type == ixgbe_mac_82599_vf) {
1306                 int msix_ctrl;
1307                 pci_find_cap(dev, PCIY_MSIX, &rid);
1308                 rid += PCIR_MSIX_CTRL;
1309                 msix_ctrl = pci_read_config(dev, rid, 2);
1310                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1311                 pci_write_config(dev, rid, msix_ctrl, 2);
1312         }
1313
1314         return (0);
1315 }
1316
1317 /*
1318  * Setup MSIX resources, note that the VF
1319  * device MUST use MSIX, there is no fallback.
1320  */
1321 static int
1322 ixv_setup_msix(struct adapter *adapter)
1323 {
1324         device_t dev = adapter->dev;
1325         int rid, want, msgs;
1326
1327
1328         /* Must have at least 2 MSIX vectors */
1329         msgs = pci_msix_count(dev);
1330         if (msgs < 2)
1331                 goto out;
1332         rid = PCIR_BAR(3);
1333         adapter->msix_mem = bus_alloc_resource_any(dev,
1334             SYS_RES_MEMORY, &rid, RF_ACTIVE);
1335         if (adapter->msix_mem == NULL) {
1336                 device_printf(adapter->dev,
1337                     "Unable to map MSIX table \n");
1338                 goto out;
1339         }
1340
1341         /*
1342         ** Want vectors for the queues,
1343         ** plus an additional for mailbox.
1344         */
1345         want = adapter->num_queues + 1;
1346         if (want > msgs) {
1347                 want = msgs;
1348                 adapter->num_queues = msgs - 1;
1349         } else
1350                 msgs = want;
1351         if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
1352                 device_printf(adapter->dev,
1353                     "Using MSIX interrupts with %d vectors\n", want);
1354                 return (want);
1355         }
1356         /* Release in case alloc was insufficient */
1357         pci_release_msi(dev);
1358 out:
1359         if (adapter->msix_mem != NULL) {
1360                 bus_release_resource(dev, SYS_RES_MEMORY,
1361                     rid, adapter->msix_mem);
1362                 adapter->msix_mem = NULL;
1363         }
1364         device_printf(adapter->dev,"MSIX config error\n");
1365         return (ENXIO);
1366 }
1367
1368
1369 static int
1370 ixv_allocate_pci_resources(struct adapter *adapter)
1371 {
1372         int             rid;
1373         device_t        dev = adapter->dev;
1374
1375         rid = PCIR_BAR(0);
1376         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1377             &rid, RF_ACTIVE);
1378
1379         if (!(adapter->pci_mem)) {
1380                 device_printf(dev,"Unable to allocate bus resource: memory\n");
1381                 return (ENXIO);
1382         }
1383
1384         adapter->osdep.mem_bus_space_tag =
1385                 rman_get_bustag(adapter->pci_mem);
1386         adapter->osdep.mem_bus_space_handle =
1387                 rman_get_bushandle(adapter->pci_mem);
1388         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
1389
1390         /* Pick up the tuneable queues */
1391         adapter->num_queues = ixv_num_queues;
1392
1393         adapter->hw.back = &adapter->osdep;
1394
1395         /*
1396         ** Now setup MSI/X, should
1397         ** return us the number of
1398         ** configured vectors.
1399         */
1400         adapter->msix = ixv_setup_msix(adapter);
1401         if (adapter->msix == ENXIO)
1402                 return (ENXIO);
1403         else
1404                 return (0);
1405 }
1406
1407 static void
1408 ixv_free_pci_resources(struct adapter * adapter)
1409 {
1410         struct          ix_queue *que = adapter->queues;
1411         device_t        dev = adapter->dev;
1412         int             rid, memrid;
1413
1414         memrid = PCIR_BAR(MSIX_82598_BAR);
1415
1416         /*
1417         ** There is a slight possibility of a failure mode
1418         ** in attach that will result in entering this function
1419         ** before interrupt resources have been initialized, and
1420         ** in that case we do not want to execute the loops below
1421         ** We can detect this reliably by the state of the adapter
1422         ** res pointer.
1423         */
1424         if (adapter->res == NULL)
1425                 goto mem;
1426
1427         /*
1428         **  Release all msix queue resources:
1429         */
1430         for (int i = 0; i < adapter->num_queues; i++, que++) {
1431                 rid = que->msix + 1;
1432                 if (que->tag != NULL) {
1433                         bus_teardown_intr(dev, que->res, que->tag);
1434                         que->tag = NULL;
1435                 }
1436                 if (que->res != NULL)
1437                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
1438         }
1439
1440
1441         /* Clean the Legacy or Link interrupt last */
1442         if (adapter->vector) /* we are doing MSIX */
1443                 rid = adapter->vector + 1;
1444         else
1445                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
1446
1447         if (adapter->tag != NULL) {
1448                 bus_teardown_intr(dev, adapter->res, adapter->tag);
1449                 adapter->tag = NULL;
1450         }
1451         if (adapter->res != NULL)
1452                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
1453
1454 mem:
1455         if (adapter->msix)
1456                 pci_release_msi(dev);
1457
1458         if (adapter->msix_mem != NULL)
1459                 bus_release_resource(dev, SYS_RES_MEMORY,
1460                     memrid, adapter->msix_mem);
1461
1462         if (adapter->pci_mem != NULL)
1463                 bus_release_resource(dev, SYS_RES_MEMORY,
1464                     PCIR_BAR(0), adapter->pci_mem);
1465
1466         return;
1467 }
1468
1469 /*********************************************************************
1470  *
1471  *  Setup networking device structure and register an interface.
1472  *
1473  **********************************************************************/
1474 static void
1475 ixv_setup_interface(device_t dev, struct adapter *adapter)
1476 {
1477         struct ifnet   *ifp;
1478
1479         INIT_DEBUGOUT("ixv_setup_interface: begin");
1480
1481         ifp = adapter->ifp = if_alloc(IFT_ETHER);
1482         if (ifp == NULL)
1483                 panic("%s: can not if_alloc()\n", device_get_nameunit(dev));
1484         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1485         ifp->if_baudrate = 1000000000;
1486         ifp->if_init = ixv_init;
1487         ifp->if_softc = adapter;
1488         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1489         ifp->if_ioctl = ixv_ioctl;
1490 #if __FreeBSD_version >= 800000
1491         ifp->if_transmit = ixgbe_mq_start;
1492         ifp->if_qflush = ixgbe_qflush;
1493 #else
1494         ifp->if_start = ixgbe_start;
1495 #endif
1496         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
1497
1498         ether_ifattach(ifp, adapter->hw.mac.addr);
1499
1500         adapter->max_frame_size =
1501             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1502
1503         /*
1504          * Tell the upper layer(s) we support long frames.
1505          */
1506         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
1507
1508         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM;
1509         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
1510         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
1511                              |  IFCAP_VLAN_HWTSO
1512                              |  IFCAP_VLAN_MTU;
1513         ifp->if_capabilities |= IFCAP_LRO;
1514         ifp->if_capenable = ifp->if_capabilities;
1515
1516         /*
1517          * Specify the media types supported by this adapter and register
1518          * callbacks to update media and link information
1519          */
1520         ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change,
1521                      ixv_media_status);
1522         ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL);
1523         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1524         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1525
1526         return;
1527 }
1528
1529 static void
1530 ixv_config_link(struct adapter *adapter)
1531 {
1532         struct ixgbe_hw *hw = &adapter->hw;
1533         u32     autoneg, err = 0;
1534
1535         if (hw->mac.ops.check_link)
1536                 err = hw->mac.ops.check_link(hw, &autoneg,
1537                     &adapter->link_up, FALSE);
1538         if (err)
1539                 goto out;
1540
1541         if (hw->mac.ops.setup_link)
1542                 err = hw->mac.ops.setup_link(hw,
1543                     autoneg, adapter->link_up);
1544 out:
1545         return;
1546 }
1547
1548
1549 /*********************************************************************
1550  *
1551  *  Enable transmit unit.
1552  *
1553  **********************************************************************/
1554 static void
1555 ixv_initialize_transmit_units(struct adapter *adapter)
1556 {
1557         struct tx_ring  *txr = adapter->tx_rings;
1558         struct ixgbe_hw *hw = &adapter->hw;
1559
1560
1561         for (int i = 0; i < adapter->num_queues; i++, txr++) {
1562                 u64     tdba = txr->txdma.dma_paddr;
1563                 u32     txctrl, txdctl;
1564
1565                 /* Set WTHRESH to 8, burst writeback */
1566                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
1567                 txdctl |= (8 << 16);
1568                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
1569
1570                 /* Set the HW Tx Head and Tail indices */
1571                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0);
1572                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0);
1573
1574                 /* Set Tx Tail register */
1575                 txr->tail = IXGBE_VFTDT(i);
1576
1577                 /* Set the processing limit */
1578                 txr->process_limit = ixv_tx_process_limit;
1579
1580                 /* Set Ring parameters */
1581                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
1582                        (tdba & 0x00000000ffffffffULL));
1583                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32));
1584                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
1585                     adapter->num_tx_desc *
1586                     sizeof(struct ixgbe_legacy_tx_desc));
1587                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i));
1588                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
1589                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl);
1590
1591                 /* Now enable */
1592                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
1593                 txdctl |= IXGBE_TXDCTL_ENABLE;
1594                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
1595         }
1596
1597         return;
1598 }
1599
1600
1601 /*********************************************************************
1602  *
1603  *  Setup receive registers and features.
1604  *
1605  **********************************************************************/
1606 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1607
1608 static void
1609 ixv_initialize_receive_units(struct adapter *adapter)
1610 {
1611         struct  rx_ring *rxr = adapter->rx_rings;
1612         struct ixgbe_hw *hw = &adapter->hw;
1613         struct ifnet    *ifp = adapter->ifp;
1614         u32             bufsz, rxcsum, psrtype;
1615         int             max_frame;
1616
1617         if (ifp->if_mtu > ETHERMTU)
1618                 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1619         else
1620                 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1621
1622         psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1623             IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1624             IXGBE_PSRTYPE_L2HDR;
1625
1626         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1627
1628         /* Tell PF our expected packet-size */
1629         max_frame = ifp->if_mtu + IXGBE_MTU_HDR;
1630         ixgbevf_rlpml_set_vf(hw, max_frame);
1631
1632         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
1633                 u64 rdba = rxr->rxdma.dma_paddr;
1634                 u32 reg, rxdctl;
1635
1636                 /* Disable the queue */
1637                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
1638                 rxdctl &= ~(IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME);
1639                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
1640                 for (int j = 0; j < 10; j++) {
1641                         if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) &
1642                             IXGBE_RXDCTL_ENABLE)
1643                                 msec_delay(1);
1644                         else
1645                                 break;
1646                 }
1647                 wmb();
1648                 /* Setup the Base and Length of the Rx Descriptor Ring */
1649                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
1650                     (rdba & 0x00000000ffffffffULL));
1651                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
1652                     (rdba >> 32));
1653                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
1654                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
1655
1656                 /* Reset the ring indices */
1657                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
1658                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0);
1659
1660                 /* Set up the SRRCTL register */
1661                 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i));
1662                 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
1663                 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
1664                 reg |= bufsz;
1665                 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1666                 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg);
1667
1668                 /* Set the Tail Pointer */
1669                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
1670                     adapter->num_rx_desc - 1);
1671
1672                 /* Set the processing limit */
1673                 rxr->process_limit = ixv_rx_process_limit;
1674
1675                 /* Capture Rx Tail index */
1676                 rxr->tail = IXGBE_VFRDT(rxr->me);
1677
1678                 /* Do the queue enabling last */
1679                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
1680                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1681                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
1682                 for (int k = 0; k < 10; k++) {
1683                         if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) &
1684                             IXGBE_RXDCTL_ENABLE)
1685                                 break;
1686                         else
1687                                 msec_delay(1);
1688                 }
1689                 wmb();
1690         }
1691
1692         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
1693
1694         if (ifp->if_capenable & IFCAP_RXCSUM)
1695                 rxcsum |= IXGBE_RXCSUM_PCSD;
1696
1697         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
1698                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
1699
1700         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
1701
1702         return;
1703 }
1704
1705 static void
1706 ixv_setup_vlan_support(struct adapter *adapter)
1707 {
1708         struct ixgbe_hw *hw = &adapter->hw;
1709         u32             ctrl, vid, vfta, retry;
1710
1711
1712         /*
1713         ** We get here thru init_locked, meaning
1714         ** a soft reset, this has already cleared
1715         ** the VFTA and other state, so if there
1716         ** have been no vlan's registered do nothing.
1717         */
1718         if (adapter->num_vlans == 0)
1719                 return;
1720
1721         /* Enable the queues */
1722         for (int i = 0; i < adapter->num_queues; i++) {
1723                 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
1724                 ctrl |= IXGBE_RXDCTL_VME;
1725                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl);
1726         }
1727
1728         /*
1729         ** A soft reset zero's out the VFTA, so
1730         ** we need to repopulate it now.
1731         */
1732         for (int i = 0; i < IXGBE_VFTA_SIZE; i++) {
1733                 if (ixv_shadow_vfta[i] == 0)
1734                         continue;
1735                 vfta = ixv_shadow_vfta[i];
1736                 /*
1737                 ** Reconstruct the vlan id's
1738                 ** based on the bits set in each
1739                 ** of the array ints.
1740                 */
1741                 for ( int j = 0; j < 32; j++) {
1742                         retry = 0;
1743                         if ((vfta & (1 << j)) == 0)
1744                                 continue;
1745                         vid = (i * 32) + j;
1746                         /* Call the shared code mailbox routine */
1747                         while (ixgbe_set_vfta(hw, vid, 0, TRUE)) {
1748                                 if (++retry > 5)
1749                                         break;
1750                         }
1751                 }
1752         }
1753 }
1754
1755 /*
1756 ** This routine is run via an vlan config EVENT,
1757 ** it enables us to use the HW Filter table since
1758 ** we can get the vlan id. This just creates the
1759 ** entry in the soft version of the VFTA, init will
1760 ** repopulate the real table.
1761 */
1762 static void
1763 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
1764 {
1765         struct adapter  *adapter = ifp->if_softc;
1766         u16             index, bit;
1767
1768         if (ifp->if_softc !=  arg)   /* Not our event */
1769                 return;
1770
1771         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
1772                 return;
1773
1774         IXGBE_CORE_LOCK(adapter);
1775         index = (vtag >> 5) & 0x7F;
1776         bit = vtag & 0x1F;
1777         ixv_shadow_vfta[index] |= (1 << bit);
1778         ++adapter->num_vlans;
1779         /* Re-init to load the changes */
1780         ixv_init_locked(adapter);
1781         IXGBE_CORE_UNLOCK(adapter);
1782 }
1783
1784 /*
1785 ** This routine is run via an vlan
1786 ** unconfig EVENT, remove our entry
1787 ** in the soft vfta.
1788 */
1789 static void
1790 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
1791 {
1792         struct adapter  *adapter = ifp->if_softc;
1793         u16             index, bit;
1794
1795         if (ifp->if_softc !=  arg)
1796                 return;
1797
1798         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
1799                 return;
1800
1801         IXGBE_CORE_LOCK(adapter);
1802         index = (vtag >> 5) & 0x7F;
1803         bit = vtag & 0x1F;
1804         ixv_shadow_vfta[index] &= ~(1 << bit);
1805         --adapter->num_vlans;
1806         /* Re-init to load the changes */
1807         ixv_init_locked(adapter);
1808         IXGBE_CORE_UNLOCK(adapter);
1809 }
1810
1811 static void
1812 ixv_enable_intr(struct adapter *adapter)
1813 {
1814         struct ixgbe_hw *hw = &adapter->hw;
1815         struct ix_queue *que = adapter->queues;
1816         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1817
1818
1819         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
1820
1821         mask = IXGBE_EIMS_ENABLE_MASK;
1822         mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1823         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
1824
1825         for (int i = 0; i < adapter->num_queues; i++, que++)
1826                 ixv_enable_queue(adapter, que->msix);
1827
1828         IXGBE_WRITE_FLUSH(hw);
1829
1830         return;
1831 }
1832
1833 static void
1834 ixv_disable_intr(struct adapter *adapter)
1835 {
1836         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0);
1837         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0);
1838         IXGBE_WRITE_FLUSH(&adapter->hw);
1839         return;
1840 }
1841
1842 /*
1843 ** Setup the correct IVAR register for a particular MSIX interrupt
1844 **  - entry is the register array entry
1845 **  - vector is the MSIX vector for this queue
1846 **  - type is RX/TX/MISC
1847 */
1848 static void
1849 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
1850 {
1851         struct ixgbe_hw *hw = &adapter->hw;
1852         u32 ivar, index;
1853
1854         vector |= IXGBE_IVAR_ALLOC_VAL;
1855
1856         if (type == -1) { /* MISC IVAR */
1857                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
1858                 ivar &= ~0xFF;
1859                 ivar |= vector;
1860                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
1861         } else {        /* RX/TX IVARS */
1862                 index = (16 * (entry & 1)) + (8 * type);
1863                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
1864                 ivar &= ~(0xFF << index);
1865                 ivar |= (vector << index);
1866                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
1867         }
1868 }
1869
1870 static void
1871 ixv_configure_ivars(struct adapter *adapter)
1872 {
1873         struct  ix_queue *que = adapter->queues;
1874
1875         for (int i = 0; i < adapter->num_queues; i++, que++) {
1876                 /* First the RX queue entry */
1877                 ixv_set_ivar(adapter, i, que->msix, 0);
1878                 /* ... and the TX */
1879                 ixv_set_ivar(adapter, i, que->msix, 1);
1880                 /* Set an initial value in EITR */
1881                 IXGBE_WRITE_REG(&adapter->hw,
1882                     IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT);
1883         }
1884
1885         /* For the mailbox interrupt */
1886         ixv_set_ivar(adapter, 1, adapter->vector, -1);
1887 }
1888
1889
1890 /*
1891 ** Tasklet handler for MSIX MBX interrupts
1892 **  - do outside interrupt since it might sleep
1893 */
1894 static void
1895 ixv_handle_mbx(void *context, int pending)
1896 {
1897         struct adapter  *adapter = context;
1898
1899         ixgbe_check_link(&adapter->hw,
1900             &adapter->link_speed, &adapter->link_up, 0);
1901         ixv_update_link_status(adapter);
1902 }
1903
1904 /*
1905 ** The VF stats registers never have a truely virgin
1906 ** starting point, so this routine tries to make an
1907 ** artificial one, marking ground zero on attach as
1908 ** it were.
1909 */
1910 static void
1911 ixv_save_stats(struct adapter *adapter)
1912 {
1913         if (adapter->stats.vf.vfgprc || adapter->stats.vf.vfgptc) {
1914                 adapter->stats.vf.saved_reset_vfgprc +=
1915                     adapter->stats.vf.vfgprc - adapter->stats.vf.base_vfgprc;
1916                 adapter->stats.vf.saved_reset_vfgptc +=
1917                     adapter->stats.vf.vfgptc - adapter->stats.vf.base_vfgptc;
1918                 adapter->stats.vf.saved_reset_vfgorc +=
1919                     adapter->stats.vf.vfgorc - adapter->stats.vf.base_vfgorc;
1920                 adapter->stats.vf.saved_reset_vfgotc +=
1921                     adapter->stats.vf.vfgotc - adapter->stats.vf.base_vfgotc;
1922                 adapter->stats.vf.saved_reset_vfmprc +=
1923                     adapter->stats.vf.vfmprc - adapter->stats.vf.base_vfmprc;
1924         }
1925 }
1926
1927 static void
1928 ixv_init_stats(struct adapter *adapter)
1929 {
1930         struct ixgbe_hw *hw = &adapter->hw;
1931
1932         adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1933         adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1934         adapter->stats.vf.last_vfgorc |=
1935             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1936
1937         adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1938         adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1939         adapter->stats.vf.last_vfgotc |=
1940             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1941
1942         adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1943
1944         adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc;
1945         adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc;
1946         adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc;
1947         adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc;
1948         adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc;
1949 }
1950
1951 #define UPDATE_STAT_32(reg, last, count)                \
1952 {                                                       \
1953         u32 current = IXGBE_READ_REG(hw, reg);          \
1954         if (current < last)                             \
1955                 count += 0x100000000LL;                 \
1956         last = current;                                 \
1957         count &= 0xFFFFFFFF00000000LL;                  \
1958         count |= current;                               \
1959 }
1960
1961 #define UPDATE_STAT_36(lsb, msb, last, count)           \
1962 {                                                       \
1963         u64 cur_lsb = IXGBE_READ_REG(hw, lsb);          \
1964         u64 cur_msb = IXGBE_READ_REG(hw, msb);          \
1965         u64 current = ((cur_msb << 32) | cur_lsb);      \
1966         if (current < last)                             \
1967                 count += 0x1000000000LL;                \
1968         last = current;                                 \
1969         count &= 0xFFFFFFF000000000LL;                  \
1970         count |= current;                               \
1971 }
1972
1973 /*
1974 ** ixv_update_stats - Update the board statistics counters.
1975 */
1976 void
1977 ixv_update_stats(struct adapter *adapter)
1978 {
1979         struct ixgbe_hw *hw = &adapter->hw;
1980
1981         UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.vf.last_vfgprc,
1982             adapter->stats.vf.vfgprc);
1983         UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.vf.last_vfgptc,
1984             adapter->stats.vf.vfgptc);
1985         UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1986             adapter->stats.vf.last_vfgorc, adapter->stats.vf.vfgorc);
1987         UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1988             adapter->stats.vf.last_vfgotc, adapter->stats.vf.vfgotc);
1989         UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.vf.last_vfmprc,
1990             adapter->stats.vf.vfmprc);
1991 }
1992
1993 /*
1994  * Add statistic sysctls for the VF.
1995  */
1996 static void
1997 ixv_add_stats_sysctls(struct adapter *adapter)
1998 {
1999         device_t dev = adapter->dev;
2000         struct ix_queue *que = &adapter->queues[0];
2001         struct tx_ring *txr = que->txr;
2002         struct rx_ring *rxr = que->rxr;
2003
2004         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
2005         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
2006         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
2007         struct ixgbevf_hw_stats *stats = &adapter->stats.vf;
2008
2009         struct sysctl_oid *stat_node, *queue_node;
2010         struct sysctl_oid_list *stat_list, *queue_list;
2011
2012         /* Driver Statistics */
2013         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
2014                         CTLFLAG_RD, &adapter->dropped_pkts,
2015                         "Driver dropped packets");
2016         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
2017                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
2018                         "m_defrag() failed");
2019         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
2020                         CTLFLAG_RD, &adapter->watchdog_events,
2021                         "Watchdog timeouts");
2022
2023         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
2024                                     CTLFLAG_RD, NULL,
2025                                     "VF Statistics (read from HW registers)");
2026         stat_list = SYSCTL_CHILDREN(stat_node);
2027
2028         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
2029                         CTLFLAG_RD, &stats->vfgprc,
2030                         "Good Packets Received");
2031         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
2032                         CTLFLAG_RD, &stats->vfgorc,
2033                         "Good Octets Received");
2034         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
2035                         CTLFLAG_RD, &stats->vfmprc,
2036                         "Multicast Packets Received");
2037         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
2038                         CTLFLAG_RD, &stats->vfgptc,
2039                         "Good Packets Transmitted");
2040         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
2041                         CTLFLAG_RD, &stats->vfgotc,
2042                         "Good Octets Transmitted");
2043
2044         queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "que",
2045                                     CTLFLAG_RD, NULL,
2046                                     "Queue Statistics (collected by SW)");
2047         queue_list = SYSCTL_CHILDREN(queue_node);
2048
2049         SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
2050                         CTLFLAG_RD, &(que->irqs),
2051                         "IRQs on queue");
2052         SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_irqs",
2053                         CTLFLAG_RD, &(rxr->rx_irq),
2054                         "RX irqs on queue");
2055         SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
2056                         CTLFLAG_RD, &(rxr->rx_packets),
2057                         "RX packets");
2058         SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
2059                         CTLFLAG_RD, &(rxr->rx_bytes),
2060                         "RX bytes");
2061         SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
2062                         CTLFLAG_RD, &(rxr->rx_discarded),
2063                         "Discarded RX packets");
2064
2065         SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
2066                         CTLFLAG_RD, &(txr->total_packets),
2067                         "TX Packets");
2068
2069         SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_no_desc",
2070                         CTLFLAG_RD, &(txr->no_desc_avail),
2071                         "# of times not enough descriptors were available during TX");
2072 }
2073
2074 /**********************************************************************
2075  *
2076  *  This routine is called only when em_display_debug_stats is enabled.
2077  *  This routine provides a way to take a look at important statistics
2078  *  maintained by the driver and hardware.
2079  *
2080  **********************************************************************/
2081 static void
2082 ixv_print_debug_info(struct adapter *adapter)
2083 {
2084         device_t dev = adapter->dev;
2085         struct ixgbe_hw         *hw = &adapter->hw;
2086         struct ix_queue         *que = adapter->queues;
2087         struct rx_ring          *rxr;
2088         struct tx_ring          *txr;
2089         struct lro_ctrl         *lro;
2090
2091         device_printf(dev,"Error Byte Count = %u \n",
2092             IXGBE_READ_REG(hw, IXGBE_ERRBC));
2093
2094         for (int i = 0; i < adapter->num_queues; i++, que++) {
2095                 txr = que->txr;
2096                 rxr = que->rxr;
2097                 lro = &rxr->lro;
2098                 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n",
2099                     que->msix, (long)que->irqs);
2100                 device_printf(dev,"RX(%d) Packets Received: %lld\n",
2101                     rxr->me, (long long)rxr->rx_packets);
2102                 device_printf(dev,"RX(%d) Bytes Received: %lu\n",
2103                     rxr->me, (long)rxr->rx_bytes);
2104                 device_printf(dev,"RX(%d) LRO Queued= %d\n",
2105                     rxr->me, lro->lro_queued);
2106                 device_printf(dev,"RX(%d) LRO Flushed= %d\n",
2107                     rxr->me, lro->lro_flushed);
2108                 device_printf(dev,"TX(%d) Packets Sent: %lu\n",
2109                     txr->me, (long)txr->total_packets);
2110                 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n",
2111                     txr->me, (long)txr->no_desc_avail);
2112         }
2113
2114         device_printf(dev,"MBX IRQ Handled: %lu\n",
2115             (long)adapter->link_irq);
2116         return;
2117 }
2118
2119 static int
2120 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)
2121 {
2122         int error, result;
2123         struct adapter *adapter;
2124
2125         result = -1;
2126         error = sysctl_handle_int(oidp, &result, 0, req);
2127
2128         if (error || !req->newptr)
2129                 return (error);
2130
2131         if (result == 1) {
2132                 adapter = (struct adapter *) arg1;
2133                 ixv_print_debug_info(adapter);
2134         }
2135         return error;
2136 }
2137