sys/dev/ixgbe/if_ix.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #ifndef IXGBE_STANDALONE_BUILD
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #include "opt_rss.h"
  40 #endif
  41
  42 #include "ixgbe.h"
  43
  44 #ifdef  RSS
  45 #include <net/rss_config.h>
  46 #include <netinet/in_rss.h>
  47 #endif
  48
  49 /*********************************************************************
  50  *  Set this to one to display debug statistics
  51  *********************************************************************/
  52 int             ixgbe_display_debug_stats = 0;
  53
  54 /*********************************************************************
  55  *  Driver version
  56  *********************************************************************/
  57 char ixgbe_driver_version[] = "2.7.4";
  58
  59 /*********************************************************************
  60  *  PCI Device ID Table
  61  *
  62  *  Used by probe to select devices to load on
  63  *  Last field stores an index into ixgbe_strings
  64  *  Last entry must be all 0s
  65  *
  66  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  67  *********************************************************************/
  68
  69 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  70 {
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  86         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  87         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  88         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  89         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  90         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  91         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  92         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  93         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
  94         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0},
  95         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  96         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  97         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0},
  98         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0},
  99         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0},
 100         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0},
 101         /* required last entry */
 102         {0, 0, 0, 0, 0}
 103 };
 104
 105 /*********************************************************************
 106  *  Table of branding strings
 107  *********************************************************************/
 108
 109 static char    *ixgbe_strings[] = {
 110         "Intel(R) PRO/10GbE PCI-Express Network Driver"
 111 };
 112
 113 /*********************************************************************
 114  *  Function prototypes
 115  *********************************************************************/
 116 static int      ixgbe_probe(device_t);
 117 static int      ixgbe_attach(device_t);
 118 static int      ixgbe_detach(device_t);
 119 static int      ixgbe_shutdown(device_t);
 120 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 121 static void     ixgbe_init(void *);
 122 static void     ixgbe_init_locked(struct adapter *);
 123 static void     ixgbe_stop(void *);
 124 #if __FreeBSD_version >= 1100036
 125 static uint64_t ixgbe_get_counter(struct ifnet *, ift_counter);
 126 #endif
 127 static void     ixgbe_add_media_types(struct adapter *);
 128 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 129 static int      ixgbe_media_change(struct ifnet *);
 130 static void     ixgbe_identify_hardware(struct adapter *);
 131 static int      ixgbe_allocate_pci_resources(struct adapter *);
 132 static void     ixgbe_get_slot_info(struct ixgbe_hw *);
 133 static int      ixgbe_allocate_msix(struct adapter *);
 134 static int      ixgbe_allocate_legacy(struct adapter *);
 135 static int      ixgbe_setup_msix(struct adapter *);
 136 static void     ixgbe_free_pci_resources(struct adapter *);
 137 static void     ixgbe_local_timer(void *);
 138 static int      ixgbe_setup_interface(device_t, struct adapter *);
 139 static void     ixgbe_config_link(struct adapter *);
 140 static void     ixgbe_rearm_queues(struct adapter *, u64);
 141
 142 static void     ixgbe_initialize_transmit_units(struct adapter *);
 143 static void     ixgbe_initialize_receive_units(struct adapter *);
 144 static void     ixgbe_enable_rx_drop(struct adapter *);
 145 static void     ixgbe_disable_rx_drop(struct adapter *);
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_set_promisc(struct adapter *);
 151 static void     ixgbe_set_multi(struct adapter *);
 152 static void     ixgbe_update_link_status(struct adapter *);
 153 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 154 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 155 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 156 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 157 static void     ixgbe_configure_ivars(struct adapter *);
 158 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 159
 160 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 161 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 162 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 163
 164 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 165
 166 /* Support for pluggable optic modules */
 167 static bool     ixgbe_sfp_probe(struct adapter *);
 168 static void     ixgbe_setup_optics(struct adapter *);
 169
 170 /* Legacy (single vector interrupt handler */
 171 static void     ixgbe_legacy_irq(void *);
 172
 173 /* The MSI/X Interrupt handlers */
 174 static void     ixgbe_msix_que(void *);
 175 static void     ixgbe_msix_link(void *);
 176
 177 /* Deferred interrupt tasklets */
 178 static void     ixgbe_handle_que(void *, int);
 179 static void     ixgbe_handle_link(void *, int);
 180 static void     ixgbe_handle_msf(void *, int);
 181 static void     ixgbe_handle_mod(void *, int);
 182
 183 #ifdef IXGBE_FDIR
 184 static void     ixgbe_reinit_fdir(void *, int);
 185 #endif
 186
 187
 188 /* Missing shared code prototype */
 189 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
 190
 191 /*********************************************************************
 192  *  FreeBSD Device Interface Entry Points
 193  *********************************************************************/
 194
 195 static device_method_t ix_methods[] = {
 196         /* Device interface */
 197         DEVMETHOD(device_probe, ixgbe_probe),
 198         DEVMETHOD(device_attach, ixgbe_attach),
 199         DEVMETHOD(device_detach, ixgbe_detach),
 200         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 201         DEVMETHOD_END
 202 };
 203
 204 static driver_t ix_driver = {
 205         "ix", ix_methods, sizeof(struct adapter),
 206 };
 207
 208 devclass_t ix_devclass;
 209 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0);
 210
 211 MODULE_DEPEND(ix, pci, 1, 1, 1);
 212 MODULE_DEPEND(ix, ether, 1, 1, 1);
 213
 214 /*
 215 ** TUNEABLE PARAMETERS:
 216 */
 217
 218 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD, 0,
 219                    "IXGBE driver parameters");
 220
 221 /*
 222 ** AIM: Adaptive Interrupt Moderation
 223 ** which means that the interrupt rate
 224 ** is varied over time based on the
 225 ** traffic for that interrupt vector
 226 */
 227 static int ixgbe_enable_aim = TRUE;
 228 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &ixgbe_enable_aim, 0,
 229     "Enable adaptive interrupt moderation");
 230
 231 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 232 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
 233     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
 234
 235 /* How many packets rxeof tries to clean at a time */
 236 static int ixgbe_rx_process_limit = 256;
 237 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 238 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
 239     &ixgbe_rx_process_limit, 0,
 240     "Maximum number of received packets to process at a time,"
 241     "-1 means unlimited");
 242
 243 /* How many packets txeof tries to clean at a time */
 244 static int ixgbe_tx_process_limit = 256;
 245 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
 246 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
 247     &ixgbe_tx_process_limit, 0,
 248     "Maximum number of sent packets to process at a time,"
 249     "-1 means unlimited");
 250
 251 /*
 252 ** Smart speed setting, default to on
 253 ** this only works as a compile option
 254 ** right now as its during attach, set
 255 ** this to 'ixgbe_smart_speed_off' to
 256 ** disable.
 257 */
 258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 259
 260 /*
 261  * MSIX should be the default for best performance,
 262  * but this allows it to be forced off for testing.
 263  */
 264 static int ixgbe_enable_msix = 1;
 265 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
 266     "Enable MSI-X interrupts");
 267
 268 /*
 269  * Number of Queues, can be set to 0,
 270  * it then autoconfigures based on the
 271  * number of cpus with a max of 8. This
 272  * can be overriden manually here.
 273  */
 274 static int ixgbe_num_queues = 0;
 275 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
 276     "Number of queues to configure, 0 indicates autoconfigure");
 277
 278 /*
 279 ** Number of TX descriptors per ring,
 280 ** setting higher than RX as this seems
 281 ** the better performing choice.
 282 */
 283 static int ixgbe_txd = PERFORM_TXD;
 284 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
 285     "Number of transmit descriptors per queue");
 286
 287 /* Number of RX descriptors per ring */
 288 static int ixgbe_rxd = PERFORM_RXD;
 289 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
 290     "Number of receive descriptors per queue");
 291
 292 /*
 293 ** Defining this on will allow the use
 294 ** of unsupported SFP+ modules, note that
 295 ** doing so you are on your own :)
 296 */
 297 static int allow_unsupported_sfp = FALSE;
 298 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp);
 299
 300 /* Keep running tab on them for sanity check */
 301 static int ixgbe_total_ports;
 302
 303 #ifdef IXGBE_FDIR
 304 /*
 305 ** Flow Director actually 'steals'
 306 ** part of the packet buffer as its
 307 ** filter pool, this variable controls
 308 ** how much it uses:
 309 **  0 = 64K, 1 = 128K, 2 = 256K
 310 */
 311 static int fdir_pballoc = 1;
 312 #endif
 313
 314 #ifdef DEV_NETMAP
 315 /*
 316  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 317  * be a reference on how to implement netmap support in a driver.
 318  * Additional comments are in ixgbe_netmap.h .
 319  *
 320  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 321  * that extend the standard driver.
 322  */
 323 #include <dev/netmap/ixgbe_netmap.h>
 324 #endif /* DEV_NETMAP */
 325
 326 /*********************************************************************
 327  *  Device identification routine
 328  *
 329  *  ixgbe_probe determines if the driver should be loaded on
 330  *  adapter based on PCI vendor/device id of the adapter.
 331  *
 332  *  return BUS_PROBE_DEFAULT on success, positive on failure
 333  *********************************************************************/
 334
 335 static int
 336 ixgbe_probe(device_t dev)
 337 {
 338         ixgbe_vendor_info_t *ent;
 339
 340         u16     pci_vendor_id = 0;
 341         u16     pci_device_id = 0;
 342         u16     pci_subvendor_id = 0;
 343         u16     pci_subdevice_id = 0;
 344         char    adapter_name[256];
 345
 346         INIT_DEBUGOUT("ixgbe_probe: begin");
 347
 348         pci_vendor_id = pci_get_vendor(dev);
 349         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 350                 return (ENXIO);
 351
 352         pci_device_id = pci_get_device(dev);
 353         pci_subvendor_id = pci_get_subvendor(dev);
 354         pci_subdevice_id = pci_get_subdevice(dev);
 355
 356         ent = ixgbe_vendor_info_array;
 357         while (ent->vendor_id != 0) {
 358                 if ((pci_vendor_id == ent->vendor_id) &&
 359                     (pci_device_id == ent->device_id) &&
 360
 361                     ((pci_subvendor_id == ent->subvendor_id) ||
 362                      (ent->subvendor_id == 0)) &&
 363
 364                     ((pci_subdevice_id == ent->subdevice_id) ||
 365                      (ent->subdevice_id == 0))) {
 366                         sprintf(adapter_name, "%s, Version - %s",
 367                                 ixgbe_strings[ent->index],
 368                                 ixgbe_driver_version);
 369                         device_set_desc_copy(dev, adapter_name);
 370                         ++ixgbe_total_ports;
 371                         return (BUS_PROBE_DEFAULT);
 372                 }
 373                 ent++;
 374         }
 375         return (ENXIO);
 376 }
 377
 378 /*********************************************************************
 379  *  Device initialization routine
 380  *
 381  *  The attach entry point is called when the driver is being loaded.
 382  *  This routine identifies the type of hardware, allocates all resources
 383  *  and initializes the hardware.
 384  *
 385  *  return 0 on success, positive on failure
 386  *********************************************************************/
 387
 388 static int
 389 ixgbe_attach(device_t dev)
 390 {
 391         struct adapter *adapter;
 392         struct ixgbe_hw *hw;
 393         int             error = 0;
 394         u16             csum;
 395         u32             ctrl_ext;
 396
 397         INIT_DEBUGOUT("ixgbe_attach: begin");
 398
 399         /* Allocate, clear, and link in our adapter structure */
 400         adapter = device_get_softc(dev);
 401         adapter->dev = adapter->osdep.dev = dev;
 402         hw = &adapter->hw;
 403
 404         /* Core Lock Init*/
 405         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 406
 407         /* SYSCTL APIs */
 408         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 409                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 410                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 411                         adapter, 0, ixgbe_set_flowcntl, "I", IXGBE_SYSCTL_DESC_SET_FC);
 412
 413         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 414                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 415                         OID_AUTO, "enable_aim", CTLFLAG_RW,
 416                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 417
 418         /*
 419         ** Allow a kind of speed control by forcing the autoneg
 420         ** advertised speed list to only a certain value, this
 421         ** supports 1G on 82599 devices, and 100Mb on x540.
 422         */
 423         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 424                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 425                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 426                         adapter, 0, ixgbe_set_advertise, "I", IXGBE_SYSCTL_DESC_ADV_SPEED);
 427
 428         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 429                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 430                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 431                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 432
 433         /* Set up the timer callout */
 434         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 435
 436         /* Determine hardware revision */
 437         ixgbe_identify_hardware(adapter);
 438
 439         /* Do base PCI setup - map BAR0 */
 440         if (ixgbe_allocate_pci_resources(adapter)) {
 441                 device_printf(dev, "Allocation of PCI resources failed\n");
 442                 error = ENXIO;
 443                 goto err_out;
 444         }
 445
 446         /* Do descriptor calc and sanity checks */
 447         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 448             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 449                 device_printf(dev, "TXD config issue, using default!\n");
 450                 adapter->num_tx_desc = DEFAULT_TXD;
 451         } else
 452                 adapter->num_tx_desc = ixgbe_txd;
 453
 454         /*
 455         ** With many RX rings it is easy to exceed the
 456         ** system mbuf allocation. Tuning nmbclusters
 457         ** can alleviate this.
 458         */
 459         if (nmbclusters > 0) {
 460                 int s;
 461                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 462                 if (s > nmbclusters) {
 463                         device_printf(dev, "RX Descriptors exceed "
 464                             "system mbuf max, using default instead!\n");
 465                         ixgbe_rxd = DEFAULT_RXD;
 466                 }
 467         }
 468
 469         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 470             ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) {
 471                 device_printf(dev, "RXD config issue, using default!\n");
 472                 adapter->num_rx_desc = DEFAULT_RXD;
 473         } else
 474                 adapter->num_rx_desc = ixgbe_rxd;
 475
 476         /* Allocate our TX/RX Queues */
 477         if (ixgbe_allocate_queues(adapter)) {
 478                 error = ENOMEM;
 479                 goto err_out;
 480         }
 481
 482         /* Allocate multicast array memory. */
 483         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 484             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 485         if (adapter->mta == NULL) {
 486                 device_printf(dev, "Can not allocate multicast setup array\n");
 487                 error = ENOMEM;
 488                 goto err_late;
 489         }
 490
 491         /* Initialize the shared code */
 492         hw->allow_unsupported_sfp = allow_unsupported_sfp;
 493         error = ixgbe_init_shared_code(hw);
 494         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 495                 /*
 496                 ** No optics in this port, set up
 497                 ** so the timer routine will probe
 498                 ** for later insertion.
 499                 */
 500                 adapter->sfp_probe = TRUE;
 501                 error = 0;
 502         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 503                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 504                 error = EIO;
 505                 goto err_late;
 506         } else if (error) {
 507                 device_printf(dev,"Unable to initialize the shared code\n");
 508                 error = EIO;
 509                 goto err_late;
 510         }
 511
 512         /* Make sure we have a good EEPROM before we read from it */
 513         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 514                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 515                 error = EIO;
 516                 goto err_late;
 517         }
 518
 519         error = ixgbe_init_hw(hw);
 520         switch (error) {
 521         case IXGBE_ERR_EEPROM_VERSION:
 522                 device_printf(dev, "This device is a pre-production adapter/"
 523                     "LOM.  Please be aware there may be issues associated "
 524                     "with your hardware.\n If you are experiencing problems "
 525                     "please contact your Intel or hardware representative "
 526                     "who provided you with this hardware.\n");
 527                 break;
 528         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 529                 device_printf(dev,"Unsupported SFP+ Module\n");
 530                 error = EIO;
 531                 goto err_late;
 532         case IXGBE_ERR_SFP_NOT_PRESENT:
 533                 device_printf(dev,"No SFP+ Module found\n");
 534                 /* falls thru */
 535         default:
 536                 break;
 537         }
 538
 539         /* Detect and set physical type */
 540         ixgbe_setup_optics(adapter);
 541
 542         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 543                 error = ixgbe_allocate_msix(adapter);
 544         else
 545                 error = ixgbe_allocate_legacy(adapter);
 546         if (error)
 547                 goto err_late;
 548
 549         /* Setup OS specific network interface */
 550         if (ixgbe_setup_interface(dev, adapter) != 0)
 551                 goto err_late;
 552
 553         /* Initialize statistics */
 554         ixgbe_update_stats_counters(adapter);
 555
 556         /* Register for VLAN events */
 557         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 558             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 559         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 560             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 561
 562         /*
 563         ** Check PCIE slot type/speed/width
 564         */
 565         ixgbe_get_slot_info(hw);
 566
 567
 568         /* Set an initial default flow control value */
 569         adapter->fc = ixgbe_fc_full;
 570
 571         /* let hardware know driver is loaded */
 572         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 573         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 574         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 575
 576         ixgbe_add_hw_stats(adapter);
 577
 578 #ifdef DEV_NETMAP
 579         ixgbe_netmap_attach(adapter);
 580 #endif /* DEV_NETMAP */
 581         INIT_DEBUGOUT("ixgbe_attach: end");
 582         return (0);
 583
 584 err_late:
 585         ixgbe_free_transmit_structures(adapter);
 586         ixgbe_free_receive_structures(adapter);
 587 err_out:
 588         if (adapter->ifp != NULL)
 589                 if_free(adapter->ifp);
 590         ixgbe_free_pci_resources(adapter);
 591         free(adapter->mta, M_DEVBUF);
 592         return (error);
 593 }
 594
 595 /*********************************************************************
 596  *  Device removal routine
 597  *
 598  *  The detach entry point is called when the driver is being removed.
 599  *  This routine stops the adapter and deallocates all the resources
 600  *  that were allocated for driver operation.
 601  *
 602  *  return 0 on success, positive on failure
 603  *********************************************************************/
 604
 605 static int
 606 ixgbe_detach(device_t dev)
 607 {
 608         struct adapter *adapter = device_get_softc(dev);
 609         struct ix_queue *que = adapter->queues;
 610         struct tx_ring *txr = adapter->tx_rings;
 611         u32     ctrl_ext;
 612
 613         INIT_DEBUGOUT("ixgbe_detach: begin");
 614
 615         /* Make sure VLANS are not using driver */
 616         if (adapter->ifp->if_vlantrunk != NULL) {
 617                 device_printf(dev,"Vlan in use, detach first\n");
 618                 return (EBUSY);
 619         }
 620
 621         IXGBE_CORE_LOCK(adapter);
 622         ixgbe_stop(adapter);
 623         IXGBE_CORE_UNLOCK(adapter);
 624
 625         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 626                 if (que->tq) {
 627 #ifndef IXGBE_LEGACY_TX
 628                         taskqueue_drain(que->tq, &txr->txq_task);
 629 #endif
 630                         taskqueue_drain(que->tq, &que->que_task);
 631                         taskqueue_free(que->tq);
 632                 }
 633         }
 634
 635         /* Drain the Link queue */
 636         if (adapter->tq) {
 637                 taskqueue_drain(adapter->tq, &adapter->link_task);
 638                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 639                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 640 #ifdef IXGBE_FDIR
 641                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 642 #endif
 643                 taskqueue_free(adapter->tq);
 644         }
 645
 646         /* let hardware know driver is unloading */
 647         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 648         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 649         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 650
 651         /* Unregister VLAN events */
 652         if (adapter->vlan_attach != NULL)
 653                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 654         if (adapter->vlan_detach != NULL)
 655                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 656
 657         ether_ifdetach(adapter->ifp);
 658         callout_drain(&adapter->timer);
 659 #ifdef DEV_NETMAP
 660         netmap_detach(adapter->ifp);
 661 #endif /* DEV_NETMAP */
 662         ixgbe_free_pci_resources(adapter);
 663         bus_generic_detach(dev);
 664         if_free(adapter->ifp);
 665
 666         ixgbe_free_transmit_structures(adapter);
 667         ixgbe_free_receive_structures(adapter);
 668         free(adapter->mta, M_DEVBUF);
 669
 670         IXGBE_CORE_LOCK_DESTROY(adapter);
 671         return (0);
 672 }
 673
 674 /*********************************************************************
 675  *
 676  *  Shutdown entry point
 677  *
 678  **********************************************************************/
 679
 680 static int
 681 ixgbe_shutdown(device_t dev)
 682 {
 683         struct adapter *adapter = device_get_softc(dev);
 684         IXGBE_CORE_LOCK(adapter);
 685         ixgbe_stop(adapter);
 686         IXGBE_CORE_UNLOCK(adapter);
 687         return (0);
 688 }
 689
 690
 691 /*********************************************************************
 692  *  Ioctl entry point
 693  *
 694  *  ixgbe_ioctl is called when the user wants to configure the
 695  *  interface.
 696  *
 697  *  return 0 on success, positive on failure
 698  **********************************************************************/
 699
 700 static int
 701 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 702 {
 703         struct adapter  *adapter = ifp->if_softc;
 704         struct ifreq    *ifr = (struct ifreq *) data;
 705 #if defined(INET) || defined(INET6)
 706         struct ifaddr *ifa = (struct ifaddr *)data;
 707         bool            avoid_reset = FALSE;
 708 #endif
 709         int             error = 0;
 710
 711         switch (command) {
 712
 713         case SIOCSIFADDR:
 714 #ifdef INET
 715                 if (ifa->ifa_addr->sa_family == AF_INET)
 716                         avoid_reset = TRUE;
 717 #endif
 718 #ifdef INET6
 719                 if (ifa->ifa_addr->sa_family == AF_INET6)
 720                         avoid_reset = TRUE;
 721 #endif
 722 #if defined(INET) || defined(INET6)
 723                 /*
 724                 ** Calling init results in link renegotiation,
 725                 ** so we avoid doing it when possible.
 726                 */
 727                 if (avoid_reset) {
 728                         ifp->if_flags |= IFF_UP;
 729                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 730                                 ixgbe_init(adapter);
 731                         if (!(ifp->if_flags & IFF_NOARP))
 732                                 arp_ifinit(ifp, ifa);
 733                 } else
 734                         error = ether_ioctl(ifp, command, data);
 735 #endif
 736                 break;
 737         case SIOCSIFMTU:
 738                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 739                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 740                         error = EINVAL;
 741                 } else {
 742                         IXGBE_CORE_LOCK(adapter);
 743                         ifp->if_mtu = ifr->ifr_mtu;
 744                         adapter->max_frame_size =
 745                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 746                         ixgbe_init_locked(adapter);
 747                         IXGBE_CORE_UNLOCK(adapter);
 748                 }
 749                 break;
 750         case SIOCSIFFLAGS:
 751                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 752                 IXGBE_CORE_LOCK(adapter);
 753                 if (ifp->if_flags & IFF_UP) {
 754                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 755                                 if ((ifp->if_flags ^ adapter->if_flags) &
 756                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 757                                         ixgbe_set_promisc(adapter);
 758                                 }
 759                         } else
 760                                 ixgbe_init_locked(adapter);
 761                 } else
 762                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 763                                 ixgbe_stop(adapter);
 764                 adapter->if_flags = ifp->if_flags;
 765                 IXGBE_CORE_UNLOCK(adapter);
 766                 break;
 767         case SIOCADDMULTI:
 768         case SIOCDELMULTI:
 769                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 770                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 771                         IXGBE_CORE_LOCK(adapter);
 772                         ixgbe_disable_intr(adapter);
 773                         ixgbe_set_multi(adapter);
 774                         ixgbe_enable_intr(adapter);
 775                         IXGBE_CORE_UNLOCK(adapter);
 776                 }
 777                 break;
 778         case SIOCSIFMEDIA:
 779         case SIOCGIFMEDIA:
 780                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 781                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 782                 break;
 783         case SIOCSIFCAP:
 784         {
 785                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 786                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 787                 if (mask & IFCAP_HWCSUM)
 788                         ifp->if_capenable ^= IFCAP_HWCSUM;
 789                 if (mask & IFCAP_TSO4)
 790                         ifp->if_capenable ^= IFCAP_TSO4;
 791                 if (mask & IFCAP_TSO6)
 792                         ifp->if_capenable ^= IFCAP_TSO6;
 793                 if (mask & IFCAP_LRO)
 794                         ifp->if_capenable ^= IFCAP_LRO;
 795                 if (mask & IFCAP_VLAN_HWTAGGING)
 796                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 797                 if (mask & IFCAP_VLAN_HWFILTER)
 798                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 799                 if (mask & IFCAP_VLAN_HWTSO)
 800                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 801                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 802                         IXGBE_CORE_LOCK(adapter);
 803                         ixgbe_init_locked(adapter);
 804                         IXGBE_CORE_UNLOCK(adapter);
 805                 }
 806                 VLAN_CAPABILITIES(ifp);
 807                 break;
 808         }
 809 #if __FreeBSD_version >= 1100036
 810         case SIOCGI2C:
 811         {
 812                 struct ixgbe_hw *hw = &adapter->hw;
 813                 struct ifi2creq i2c;
 814                 int i;
 815                 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
 816                 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
 817                 if (error != 0)
 818                         break;
 819                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
 820                         error = EINVAL;
 821                         break;
 822                 }
 823                 if (i2c.len > sizeof(i2c.data)) {
 824                         error = EINVAL;
 825                         break;
 826                 }
 827
 828                 for (i = 0; i < i2c.len; i++)
 829                         hw->phy.ops.read_i2c_byte(hw, i2c.offset + i,
 830                             i2c.dev_addr, &i2c.data[i]);
 831                 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
 832                 break;
 833         }
 834 #endif
 835         default:
 836                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 837                 error = ether_ioctl(ifp, command, data);
 838                 break;
 839         }
 840
 841         return (error);
 842 }
 843
 844 /*********************************************************************
 845  *  Init entry point
 846  *
 847  *  This routine is used in two ways. It is used by the stack as
 848  *  init entry point in network interface structure. It is also used
 849  *  by the driver as a hw/sw initialization routine to get to a
 850  *  consistent state.
 851  *
 852  *  return 0 on success, positive on failure
 853  **********************************************************************/
 854 #define IXGBE_MHADD_MFS_SHIFT 16
 855
 856 static void
 857 ixgbe_init_locked(struct adapter *adapter)
 858 {
 859         struct ifnet   *ifp = adapter->ifp;
 860         device_t        dev = adapter->dev;
 861         struct ixgbe_hw *hw = &adapter->hw;
 862         u32             k, txdctl, mhadd, gpie;
 863         u32             rxdctl, rxctrl;
 864
 865         mtx_assert(&adapter->core_mtx, MA_OWNED);
 866         INIT_DEBUGOUT("ixgbe_init_locked: begin");
 867         hw->adapter_stopped = FALSE;
 868         ixgbe_stop_adapter(hw);
 869         callout_stop(&adapter->timer);
 870
 871         /* reprogram the RAR[0] in case user changed it. */
 872         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
 873
 874         /* Get the latest mac address, User can use a LAA */
 875         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
 876               IXGBE_ETH_LENGTH_OF_ADDRESS);
 877         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
 878         hw->addr_ctrl.rar_used_count = 1;
 879
 880         /* Set the various hardware offload abilities */
 881         ifp->if_hwassist = 0;
 882         if (ifp->if_capenable & IFCAP_TSO)
 883                 ifp->if_hwassist |= CSUM_TSO;
 884         if (ifp->if_capenable & IFCAP_TXCSUM) {
 885                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 886 #if __FreeBSD_version >= 800000
 887                 if (hw->mac.type != ixgbe_mac_82598EB)
 888                         ifp->if_hwassist |= CSUM_SCTP;
 889 #endif
 890         }
 891
 892         /* Prepare transmit descriptors and buffers */
 893         if (ixgbe_setup_transmit_structures(adapter)) {
 894                 device_printf(dev,"Could not setup transmit structures\n");
 895                 ixgbe_stop(adapter);
 896                 return;
 897         }
 898
 899         ixgbe_init_hw(hw);
 900         ixgbe_initialize_transmit_units(adapter);
 901
 902         /* Setup Multicast table */
 903         ixgbe_set_multi(adapter);
 904
 905         /*
 906         ** Determine the correct mbuf pool
 907         ** for doing jumbo frames
 908         */
 909         if (adapter->max_frame_size <= 2048)
 910                 adapter->rx_mbuf_sz = MCLBYTES;
 911         else if (adapter->max_frame_size <= 4096)
 912                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 913         else if (adapter->max_frame_size <= 9216)
 914                 adapter->rx_mbuf_sz = MJUM9BYTES;
 915         else
 916                 adapter->rx_mbuf_sz = MJUM16BYTES;
 917
 918         /* Prepare receive descriptors and buffers */
 919         if (ixgbe_setup_receive_structures(adapter)) {
 920                 device_printf(dev,"Could not setup receive structures\n");
 921                 ixgbe_stop(adapter);
 922                 return;
 923         }
 924
 925         /* Configure RX settings */
 926         ixgbe_initialize_receive_units(adapter);
 927
 928         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
 929
 930         /* Enable Fan Failure Interrupt */
 931         gpie |= IXGBE_SDP1_GPIEN_BY_MAC(hw);
 932
 933         /* Add for Module detection */
 934         if (hw->mac.type == ixgbe_mac_82599EB)
 935                 gpie |= IXGBE_SDP2_GPIEN_BY_MAC(hw);
 936
 937         /* Thermal Failure Detection */
 938         if (hw->mac.type == ixgbe_mac_X540)
 939                 gpie |= IXGBE_SDP0_GPIEN_BY_MAC(hw);
 940
 941         if (adapter->msix > 1) {
 942                 /* Enable Enhanced MSIX mode */
 943                 gpie |= IXGBE_GPIE_MSIX_MODE;
 944                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
 945                     IXGBE_GPIE_OCD;
 946         }
 947         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
 948
 949         /* Set MTU size */
 950         if (ifp->if_mtu > ETHERMTU) {
 951                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
 952                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
 953                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
 954                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
 955         }
 956
 957         /* Now enable all the queues */
 958
 959         for (int i = 0; i < adapter->num_queues; i++) {
 960                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
 961                 txdctl |= IXGBE_TXDCTL_ENABLE;
 962                 /* Set WTHRESH to 8, burst writeback */
 963                 txdctl |= (8 << 16);
 964                 /*
 965                  * When the internal queue falls below PTHRESH (32),
 966                  * start prefetching as long as there are at least
 967                  * HTHRESH (1) buffers ready. The values are taken
 968                  * from the Intel linux driver 3.8.21.
 969                  * Prefetching enables tx line rate even with 1 queue.
 970                  */
 971                 txdctl |= (32 << 0) | (1 << 8);
 972                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
 973         }
 974
 975         for (int i = 0; i < adapter->num_queues; i++) {
 976                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 977                 if (hw->mac.type == ixgbe_mac_82598EB) {
 978                         /*
 979                         ** PTHRESH = 21
 980                         ** HTHRESH = 4
 981                         ** WTHRESH = 8
 982                         */
 983                         rxdctl &= ~0x3FFFFF;
 984                         rxdctl |= 0x080420;
 985                 }
 986                 rxdctl |= IXGBE_RXDCTL_ENABLE;
 987                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
 988                 for (k = 0; k < 10; k++) {
 989                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
 990                             IXGBE_RXDCTL_ENABLE)
 991                                 break;
 992                         else
 993                                 msec_delay(1);
 994                 }
 995                 wmb();
 996 #ifdef DEV_NETMAP
 997                 /*
 998                  * In netmap mode, we must preserve the buffers made
 999                  * available to userspace before the if_init()
1000                  * (this is true by default on the TX side, because
1001                  * init makes all buffers available to userspace).
1002                  *
1003                  * netmap_reset() and the device specific routines
1004                  * (e.g. ixgbe_setup_receive_rings()) map these
1005                  * buffers at the end of the NIC ring, so here we
1006                  * must set the RDT (tail) register to make sure
1007                  * they are not overwritten.
1008                  *
1009                  * In this driver the NIC ring starts at RDH = 0,
1010                  * RDT points to the last slot available for reception (?),
1011                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1012                  */
1013                 if (ifp->if_capenable & IFCAP_NETMAP) {
1014                         struct netmap_adapter *na = NA(adapter->ifp);
1015                         struct netmap_kring *kring = &na->rx_rings[i];
1016                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1017
1018                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1019                 } else
1020 #endif /* DEV_NETMAP */
1021                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1022         }
1023
1024         /* Enable Receive engine */
1025         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1026         if (hw->mac.type == ixgbe_mac_82598EB)
1027                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1028         rxctrl |= IXGBE_RXCTRL_RXEN;
1029         ixgbe_enable_rx_dma(hw, rxctrl);
1030
1031         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1032
1033         /* Set up MSI/X routing */
1034         if (ixgbe_enable_msix)  {
1035                 ixgbe_configure_ivars(adapter);
1036                 /* Set up auto-mask */
1037                 if (hw->mac.type == ixgbe_mac_82598EB)
1038                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1039                 else {
1040                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1041                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1042                 }
1043         } else {  /* Simple settings for Legacy/MSI */
1044                 ixgbe_set_ivar(adapter, 0, 0, 0);
1045                 ixgbe_set_ivar(adapter, 0, 0, 1);
1046                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1047         }
1048
1049 #ifdef IXGBE_FDIR
1050         /* Init Flow director */
1051         if (hw->mac.type != ixgbe_mac_82598EB) {
1052                 u32 hdrm = 32 << fdir_pballoc;
1053
1054                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1055                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1056         }
1057 #endif
1058
1059         /*
1060         ** Check on any SFP devices that
1061         ** need to be kick-started
1062         */
1063         if (hw->phy.type == ixgbe_phy_none) {
1064                 int err = hw->phy.ops.identify(hw);
1065                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1066                         device_printf(dev,
1067                             "Unsupported SFP+ module type was detected.\n");
1068                         return;
1069                 }
1070         }
1071
1072         /* Set moderation on the Link interrupt */
1073         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->vector), IXGBE_LINK_ITR);
1074
1075         /* Config/Enable Link */
1076         ixgbe_config_link(adapter);
1077
1078         /* Hardware Packet Buffer & Flow Control setup */
1079         {
1080                 u32 rxpb, frame, size, tmp;
1081
1082                 frame = adapter->max_frame_size;
1083
1084                 /* Calculate High Water */
1085                 switch (hw->mac.type) {
1086                 case ixgbe_mac_X540:
1087                 case ixgbe_mac_X550:
1088                 case ixgbe_mac_X550EM_a:
1089                 case ixgbe_mac_X550EM_x:
1090                         tmp = IXGBE_DV_X540(frame, frame);
1091                         break;
1092                 default:
1093                         tmp = IXGBE_DV(frame, frame);
1094                         break;
1095                 }
1096                 size = IXGBE_BT2KB(tmp);
1097                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1098                 hw->fc.high_water[0] = rxpb - size;
1099
1100                 /* Now calculate Low Water */
1101                 switch (hw->mac.type) {
1102                 case ixgbe_mac_X540:
1103                 case ixgbe_mac_X550:
1104                 case ixgbe_mac_X550EM_a:
1105                 case ixgbe_mac_X550EM_x:
1106                         tmp = IXGBE_LOW_DV_X540(frame);
1107                         break;
1108                 default:
1109                         tmp = IXGBE_LOW_DV(frame);
1110                         break;
1111                 }
1112                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1113
1114                 hw->fc.requested_mode = adapter->fc;
1115                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1116                 hw->fc.send_xon = TRUE;
1117         }
1118         /* Initialize the FC settings */
1119         ixgbe_start_hw(hw);
1120
1121         /* Set up VLAN support and filter */
1122         ixgbe_setup_vlan_hw_support(adapter);
1123
1124         /* And now turn on interrupts */
1125         ixgbe_enable_intr(adapter);
1126
1127         /* Now inform the stack we're ready */
1128         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1129
1130         return;
1131 }
1132
1133 static void
1134 ixgbe_init(void *arg)
1135 {
1136         struct adapter *adapter = arg;
1137
1138         IXGBE_CORE_LOCK(adapter);
1139         ixgbe_init_locked(adapter);
1140         IXGBE_CORE_UNLOCK(adapter);
1141         return;
1142 }
1143
1144
1145 /*
1146 **
1147 ** MSIX Interrupt Handlers and Tasklets
1148 **
1149 */
1150
1151 static inline void
1152 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1153 {
1154         struct ixgbe_hw *hw = &adapter->hw;
1155         u64     queue = (u64)(1 << vector);
1156         u32     mask;
1157
1158         if (hw->mac.type == ixgbe_mac_82598EB) {
1159                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1160                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1161         } else {
1162                 mask = (queue & 0xFFFFFFFF);
1163                 if (mask)
1164                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1165                 mask = (queue >> 32);
1166                 if (mask)
1167                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1168         }
1169 }
1170
1171 static inline void
1172 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1173 {
1174         struct ixgbe_hw *hw = &adapter->hw;
1175         u64     queue = (u64)(1 << vector);
1176         u32     mask;
1177
1178         if (hw->mac.type == ixgbe_mac_82598EB) {
1179                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1180                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1181         } else {
1182                 mask = (queue & 0xFFFFFFFF);
1183                 if (mask)
1184                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1185                 mask = (queue >> 32);
1186                 if (mask)
1187                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1188         }
1189 }
1190
1191 static void
1192 ixgbe_handle_que(void *context, int pending)
1193 {
1194         struct ix_queue *que = context;
1195         struct adapter  *adapter = que->adapter;
1196         struct tx_ring  *txr = que->txr;
1197         struct ifnet    *ifp = adapter->ifp;
1198
1199         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1200                 ixgbe_rxeof(que);
1201                 IXGBE_TX_LOCK(txr);
1202                 ixgbe_txeof(txr);
1203 #ifndef IXGBE_LEGACY_TX
1204                 if (!drbr_empty(ifp, txr->br))
1205                         ixgbe_mq_start_locked(ifp, txr);
1206 #else
1207                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1208                         ixgbe_start_locked(txr, ifp);
1209 #endif
1210                 IXGBE_TX_UNLOCK(txr);
1211         }
1212
1213         /* Reenable this interrupt */
1214         if (que->res != NULL)
1215                 ixgbe_enable_queue(adapter, que->msix);
1216         else
1217                 ixgbe_enable_intr(adapter);
1218         return;
1219 }
1220
1221
1222 /*********************************************************************
1223  *
1224  *  Legacy Interrupt Service routine
1225  *
1226  **********************************************************************/
1227
1228 static void
1229 ixgbe_legacy_irq(void *arg)
1230 {
1231         struct ix_queue *que = arg;
1232         struct adapter  *adapter = que->adapter;
1233         struct ixgbe_hw *hw = &adapter->hw;
1234         struct ifnet    *ifp = adapter->ifp;
1235         struct          tx_ring *txr = adapter->tx_rings;
1236         bool            more;
1237         u32             reg_eicr;
1238
1239
1240         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1241
1242         ++que->irqs;
1243         if (reg_eicr == 0) {
1244                 ixgbe_enable_intr(adapter);
1245                 return;
1246         }
1247
1248         more = ixgbe_rxeof(que);
1249
1250         IXGBE_TX_LOCK(txr);
1251         ixgbe_txeof(txr);
1252 #ifdef IXGBE_LEGACY_TX
1253         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1254                 ixgbe_start_locked(txr, ifp);
1255 #else
1256         if (!drbr_empty(ifp, txr->br))
1257                 ixgbe_mq_start_locked(ifp, txr);
1258 #endif
1259         IXGBE_TX_UNLOCK(txr);
1260
1261         /* Check for fan failure */
1262         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1263             (reg_eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
1264                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1265                     "REPLACE IMMEDIATELY!!\n");
1266                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
1267         }
1268
1269         /* Link status change */
1270         if (reg_eicr & IXGBE_EICR_LSC)
1271                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1272
1273         if (more)
1274                 taskqueue_enqueue(que->tq, &que->que_task);
1275         else
1276                 ixgbe_enable_intr(adapter);
1277         return;
1278 }
1279
1280
1281 /*********************************************************************
1282  *
1283  *  MSIX Queue Interrupt Service routine
1284  *
1285  **********************************************************************/
1286 void
1287 ixgbe_msix_que(void *arg)
1288 {
1289         struct ix_queue *que = arg;
1290         struct adapter  *adapter = que->adapter;
1291         struct ifnet    *ifp = adapter->ifp;
1292         struct tx_ring  *txr = que->txr;
1293         struct rx_ring  *rxr = que->rxr;
1294         bool            more;
1295         u32             newitr = 0;
1296
1297         /* Protect against spurious interrupts */
1298         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1299                 return;
1300
1301         ixgbe_disable_queue(adapter, que->msix);
1302         ++que->irqs;
1303
1304         more = ixgbe_rxeof(que);
1305
1306         IXGBE_TX_LOCK(txr);
1307         ixgbe_txeof(txr);
1308 #ifdef IXGBE_LEGACY_TX
1309         if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1310                 ixgbe_start_locked(txr, ifp);
1311 #else
1312         if (!drbr_empty(ifp, txr->br))
1313                 ixgbe_mq_start_locked(ifp, txr);
1314 #endif
1315         IXGBE_TX_UNLOCK(txr);
1316
1317         /* Do AIM now? */
1318
1319         if (ixgbe_enable_aim == FALSE)
1320                 goto no_calc;
1321         /*
1322         ** Do Adaptive Interrupt Moderation:
1323         **  - Write out last calculated setting
1324         **  - Calculate based on average size over
1325         **    the last interval.
1326         */
1327         if (que->eitr_setting)
1328                 IXGBE_WRITE_REG(&adapter->hw,
1329                     IXGBE_EITR(que->msix), que->eitr_setting);
1330
1331         que->eitr_setting = 0;
1332
1333         /* Idle, do nothing */
1334         if ((txr->bytes == 0) && (rxr->bytes == 0))
1335                 goto no_calc;
1336
1337         if ((txr->bytes) && (txr->packets))
1338                 newitr = txr->bytes/txr->packets;
1339         if ((rxr->bytes) && (rxr->packets))
1340                 newitr = max(newitr,
1341                     (rxr->bytes / rxr->packets));
1342         newitr += 24; /* account for hardware frame, crc */
1343
1344         /* set an upper boundary */
1345         newitr = min(newitr, 3000);
1346
1347         /* Be nice to the mid range */
1348         if ((newitr > 300) && (newitr < 1200))
1349                 newitr = (newitr / 3);
1350         else
1351                 newitr = (newitr / 2);
1352
1353         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1354                 newitr |= newitr << 16;
1355         else
1356                 newitr |= IXGBE_EITR_CNT_WDIS;
1357
1358         /* save for next interrupt */
1359         que->eitr_setting = newitr;
1360
1361         /* Reset state */
1362         txr->bytes = 0;
1363         txr->packets = 0;
1364         rxr->bytes = 0;
1365         rxr->packets = 0;
1366
1367 no_calc:
1368         if (more)
1369                 taskqueue_enqueue(que->tq, &que->que_task);
1370         else
1371                 ixgbe_enable_queue(adapter, que->msix);
1372         return;
1373 }
1374
1375
1376 static void
1377 ixgbe_msix_link(void *arg)
1378 {
1379         struct adapter  *adapter = arg;
1380         struct ixgbe_hw *hw = &adapter->hw;
1381         u32             reg_eicr;
1382
1383         ++adapter->vector_irq;
1384
1385         /* First get the cause */
1386         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1387         /* Be sure the queue bits are not cleared */
1388         reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1389         /* Clear interrupt with write */
1390         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1391
1392         /* Link status change */
1393         if (reg_eicr & IXGBE_EICR_LSC)
1394                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1395
1396         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1397 #ifdef IXGBE_FDIR
1398                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1399                         /* This is probably overkill :) */
1400                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1401                                 return;
1402                         /* Disable the interrupt */
1403                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1404                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1405                 } else
1406 #endif
1407                 if (reg_eicr & IXGBE_EICR_ECC) {
1408                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1409                             "Please Reboot!!\n");
1410                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1411                 } else
1412
1413                 if (ixgbe_is_sfp(hw)) {
1414                         if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1415                                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
1416                                 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1417                         } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1418                                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
1419                                 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1420                         }
1421                 }
1422         }
1423
1424         /* Check for fan failure */
1425         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1426             (reg_eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
1427                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1428                     "REPLACE IMMEDIATELY!!\n");
1429                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
1430         }
1431
1432         /* Check for over temp condition */
1433         switch (hw->mac.type) {
1434         case ixgbe_mac_X540:
1435         case ixgbe_mac_X550:
1436         case ixgbe_mac_X550EM_a:
1437                 if (reg_eicr & IXGBE_EICR_TS) {
1438                         device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1439                             "PHY IS SHUT DOWN!!\n");
1440                         device_printf(adapter->dev, "System shutdown required\n");
1441                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1442                 }
1443                 break;
1444         default:
1445                 /* Other MACs have no thermal sensor interrupt */
1446                 break;
1447         }
1448
1449         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1450         return;
1451 }
1452
1453 /*********************************************************************
1454  *
1455  *  Media Ioctl callback
1456  *
1457  *  This routine is called whenever the user queries the status of
1458  *  the interface using ifconfig.
1459  *
1460  **********************************************************************/
1461 static void
1462 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1463 {
1464         struct adapter *adapter = ifp->if_softc;
1465         struct ixgbe_hw *hw = &adapter->hw;
1466         int layer;
1467
1468         INIT_DEBUGOUT("ixgbe_media_status: begin");
1469         IXGBE_CORE_LOCK(adapter);
1470         ixgbe_update_link_status(adapter);
1471
1472         ifmr->ifm_status = IFM_AVALID;
1473         ifmr->ifm_active = IFM_ETHER;
1474
1475         if (!adapter->link_active) {
1476                 IXGBE_CORE_UNLOCK(adapter);
1477                 return;
1478         }
1479
1480         ifmr->ifm_status |= IFM_ACTIVE;
1481         layer = ixgbe_get_supported_physical_layer(hw);
1482
1483         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T ||
1484             layer & IXGBE_PHYSICAL_LAYER_1000BASE_T ||
1485             layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
1486                 switch (adapter->link_speed) {
1487                 case IXGBE_LINK_SPEED_10GB_FULL:
1488                         ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
1489                         break;
1490                 case IXGBE_LINK_SPEED_1GB_FULL:
1491                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1492                         break;
1493                 case IXGBE_LINK_SPEED_100_FULL:
1494                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1495                         break;
1496                 }
1497         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
1498             layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
1499                 switch (adapter->link_speed) {
1500                 case IXGBE_LINK_SPEED_10GB_FULL:
1501                         ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;
1502                         break;
1503                 }
1504         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
1505                 switch (adapter->link_speed) {
1506                 case IXGBE_LINK_SPEED_10GB_FULL:
1507                         ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
1508                         break;
1509                 case IXGBE_LINK_SPEED_1GB_FULL:
1510                         ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1511                         break;
1512                 }
1513         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM)
1514                 switch (adapter->link_speed) {
1515                 case IXGBE_LINK_SPEED_10GB_FULL:
1516                         ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;
1517                         break;
1518                 case IXGBE_LINK_SPEED_1GB_FULL:
1519                         ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1520                         break;
1521                 }
1522         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR ||
1523             layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
1524                 switch (adapter->link_speed) {
1525                 case IXGBE_LINK_SPEED_10GB_FULL:
1526                         ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1527                         break;
1528                 case IXGBE_LINK_SPEED_1GB_FULL:
1529                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1530                         break;
1531                 }
1532         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
1533                 switch (adapter->link_speed) {
1534                 case IXGBE_LINK_SPEED_10GB_FULL:
1535                         ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
1536                         break;
1537                 }
1538         /*
1539         ** XXX: These need to use the proper media types once
1540         ** they're added.
1541         */
1542         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
1543                 switch (adapter->link_speed) {
1544                 case IXGBE_LINK_SPEED_10GB_FULL:
1545                         ifmr->ifm_active |= IFM_10_T | IFM_FDX;
1546                         break;
1547                 case IXGBE_LINK_SPEED_1GB_FULL:
1548                         ifmr->ifm_active |= IFM_10_5 | IFM_FDX;
1549                         break;
1550                 }
1551         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4
1552             || layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
1553                 switch (adapter->link_speed) {
1554                 case IXGBE_LINK_SPEED_10GB_FULL:
1555                         ifmr->ifm_active |= IFM_10_2 | IFM_FDX;
1556                         break;
1557                 case IXGBE_LINK_SPEED_1GB_FULL:
1558                         ifmr->ifm_active |= IFM_10_5 | IFM_FDX;
1559                         break;
1560                 }
1561
1562         /* If nothing is recognized... */
1563         if (IFM_SUBTYPE(ifmr->ifm_active) == 0)
1564                 ifmr->ifm_active |= IFM_UNKNOWN;
1565
1566 #if __FreeBSD_version >= 900025
1567         /* Flow control setting */
1568         if (adapter->fc == ixgbe_fc_rx_pause || adapter->fc == ixgbe_fc_full)
1569                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1570         if (adapter->fc == ixgbe_fc_tx_pause || adapter->fc == ixgbe_fc_full)
1571                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1572 #endif
1573
1574         IXGBE_CORE_UNLOCK(adapter);
1575
1576         return;
1577 }
1578
1579 /*********************************************************************
1580  *
1581  *  Media Ioctl callback
1582  *
1583  *  This routine is called when the user changes speed/duplex using
1584  *  media/mediopt option with ifconfig.
1585  *
1586  **********************************************************************/
1587 static int
1588 ixgbe_media_change(struct ifnet * ifp)
1589 {
1590         struct adapter *adapter = ifp->if_softc;
1591         struct ifmedia *ifm = &adapter->media;
1592         struct ixgbe_hw *hw = &adapter->hw;
1593         ixgbe_link_speed speed = 0;
1594
1595         INIT_DEBUGOUT("ixgbe_media_change: begin");
1596
1597         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1598                 return (EINVAL);
1599
1600         /*
1601         ** We don't actually need to check against the supported
1602         ** media types of the adapter; ifmedia will take care of
1603         ** that for us.
1604         **      NOTE: this relies on falling thru the switch
1605         **      to get all the values set, it can be confusing.
1606         */
1607         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1608                 case IFM_AUTO:
1609                 case IFM_10G_T:
1610                         speed |= IXGBE_LINK_SPEED_100_FULL;
1611                 case IFM_10G_LRM:
1612                 case IFM_10G_SR:  /* KR, too */
1613                 case IFM_10G_LR:
1614                 case IFM_10G_CX4: /* KX4 for now */
1615                         speed |= IXGBE_LINK_SPEED_1GB_FULL;
1616                 case IFM_10G_TWINAX:
1617                         speed |= IXGBE_LINK_SPEED_10GB_FULL;
1618                         break;
1619                 case IFM_1000_T:
1620                         speed |= IXGBE_LINK_SPEED_100_FULL;
1621                 case IFM_1000_LX:
1622                 case IFM_1000_SX:
1623                 case IFM_1000_CX: /* KX until there's real support */
1624                         speed |= IXGBE_LINK_SPEED_1GB_FULL;
1625                         break;
1626                 case IFM_100_TX:
1627                         speed |= IXGBE_LINK_SPEED_100_FULL;
1628                         break;
1629                 default:
1630                         goto invalid;
1631         }
1632
1633         hw->mac.autotry_restart = TRUE;
1634         hw->mac.ops.setup_link(hw, speed, TRUE);
1635         adapter->advertise =
1636                 ((speed & IXGBE_LINK_SPEED_10GB_FULL) << 2) |
1637                 ((speed & IXGBE_LINK_SPEED_1GB_FULL) << 1) |
1638                 ((speed & IXGBE_LINK_SPEED_100_FULL) << 0);
1639
1640         return (0);
1641
1642 invalid:
1643         device_printf(adapter->dev, "Invalid media type\n");
1644         return (EINVAL);
1645 }
1646
1647 static void
1648 ixgbe_set_promisc(struct adapter *adapter)
1649 {
1650         u_int32_t       reg_rctl;
1651         struct ifnet   *ifp = adapter->ifp;
1652         int             mcnt = 0;
1653
1654         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1655         reg_rctl &= (~IXGBE_FCTRL_UPE);
1656         if (ifp->if_flags & IFF_ALLMULTI)
1657                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1658         else {
1659                 struct  ifmultiaddr *ifma;
1660 #if __FreeBSD_version < 800000
1661                 IF_ADDR_LOCK(ifp);
1662 #else
1663                 if_maddr_rlock(ifp);
1664 #endif
1665                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1666                         if (ifma->ifma_addr->sa_family != AF_LINK)
1667                                 continue;
1668                         if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1669                                 break;
1670                         mcnt++;
1671                 }
1672 #if __FreeBSD_version < 800000
1673                 IF_ADDR_UNLOCK(ifp);
1674 #else
1675                 if_maddr_runlock(ifp);
1676 #endif
1677         }
1678         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1679                 reg_rctl &= (~IXGBE_FCTRL_MPE);
1680         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1681
1682         if (ifp->if_flags & IFF_PROMISC) {
1683                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1684                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1685         } else if (ifp->if_flags & IFF_ALLMULTI) {
1686                 reg_rctl |= IXGBE_FCTRL_MPE;
1687                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1688                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1689         }
1690         return;
1691 }
1692
1693
1694 /*********************************************************************
1695  *  Multicast Update
1696  *
1697  *  This routine is called whenever multicast address list is updated.
1698  *
1699  **********************************************************************/
1700 #define IXGBE_RAR_ENTRIES 16
1701
1702 static void
1703 ixgbe_set_multi(struct adapter *adapter)
1704 {
1705         u32     fctrl;
1706         u8      *mta;
1707         u8      *update_ptr;
1708         struct  ifmultiaddr *ifma;
1709         int     mcnt = 0;
1710         struct ifnet   *ifp = adapter->ifp;
1711
1712         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1713
1714         mta = adapter->mta;
1715         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1716             MAX_NUM_MULTICAST_ADDRESSES);
1717
1718 #if __FreeBSD_version < 800000
1719         IF_ADDR_LOCK(ifp);
1720 #else
1721         if_maddr_rlock(ifp);
1722 #endif
1723         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1724                 if (ifma->ifma_addr->sa_family != AF_LINK)
1725                         continue;
1726                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1727                         break;
1728                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1729                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1730                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1731                 mcnt++;
1732         }
1733 #if __FreeBSD_version < 800000
1734         IF_ADDR_UNLOCK(ifp);
1735 #else
1736         if_maddr_runlock(ifp);
1737 #endif
1738
1739         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1740         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1741         if (ifp->if_flags & IFF_PROMISC)
1742                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1743         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1744             ifp->if_flags & IFF_ALLMULTI) {
1745                 fctrl |= IXGBE_FCTRL_MPE;
1746                 fctrl &= ~IXGBE_FCTRL_UPE;
1747         } else
1748                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1749
1750         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1751
1752         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
1753                 update_ptr = mta;
1754                 ixgbe_update_mc_addr_list(&adapter->hw,
1755                     update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1756         }
1757
1758         return;
1759 }
1760
1761 /*
1762  * This is an iterator function now needed by the multicast
1763  * shared code. It simply feeds the shared code routine the
1764  * addresses in the array of ixgbe_set_multi() one by one.
1765  */
1766 static u8 *
1767 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1768 {
1769         u8 *addr = *update_ptr;
1770         u8 *newptr;
1771         *vmdq = 0;
1772
1773         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1774         *update_ptr = newptr;
1775         return addr;
1776 }
1777
1778
1779 /*********************************************************************
1780  *  Timer routine
1781  *
1782  *  This routine checks for link status,updates statistics,
1783  *  and runs the watchdog check.
1784  *
1785  **********************************************************************/
1786
1787 static void
1788 ixgbe_local_timer(void *arg)
1789 {
1790         struct adapter  *adapter = arg;
1791         device_t        dev = adapter->dev;
1792         struct ix_queue *que = adapter->queues;
1793         u64             queues = 0;
1794         int             hung = 0;
1795
1796         mtx_assert(&adapter->core_mtx, MA_OWNED);
1797
1798         /* Check for pluggable optics */
1799         if (adapter->sfp_probe)
1800                 if (!ixgbe_sfp_probe(adapter))
1801                         goto out; /* Nothing to do */
1802
1803         ixgbe_update_link_status(adapter);
1804         ixgbe_update_stats_counters(adapter);
1805
1806         /*
1807         ** Check the TX queues status
1808         **      - mark hung queues so we don't schedule on them
1809         **      - watchdog only if all queues show hung
1810         */
1811         for (int i = 0; i < adapter->num_queues; i++, que++) {
1812                 /* Keep track of queues with work for soft irq */
1813                 if (que->txr->busy)
1814                         queues |= ((u64)1 << que->me);
1815                 /*
1816                 ** Each time txeof runs without cleaning, but there
1817                 ** are uncleaned descriptors it increments busy. If
1818                 ** we get to the MAX we declare it hung.
1819                 */
1820                 if (que->busy == IXGBE_QUEUE_HUNG) {
1821                         ++hung;
1822                         /* Mark the queue as inactive */
1823                         adapter->active_queues &= ~((u64)1 << que->me);
1824                         continue;
1825                 } else {
1826                         /* Check if we've come back from hung */
1827                         if ((adapter->active_queues & ((u64)1 << que->me)) == 0)
1828                                 adapter->active_queues |= ((u64)1 << que->me);
1829                 }
1830                 if (que->busy >= IXGBE_MAX_TX_BUSY) {
1831                         device_printf(dev,"Warning queue %d "
1832                             "appears to be hung!\n", i);
1833                         que->txr->busy = IXGBE_QUEUE_HUNG;
1834                         ++hung;
1835                 }
1836
1837         }
1838
1839         /* Only truly watchdog if all queues show hung */
1840         if (hung == adapter->num_queues)
1841                 goto watchdog;
1842         else if (queues != 0) { /* Force an IRQ on queues with work */
1843                 ixgbe_rearm_queues(adapter, queues);
1844         }
1845
1846 out:
1847         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1848         return;
1849
1850 watchdog:
1851         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1852         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1853         adapter->watchdog_events++;
1854         ixgbe_init_locked(adapter);
1855 }
1856
1857 /*
1858 ** Note: this routine updates the OS on the link state
1859 **      the real check of the hardware only happens with
1860 **      a link interrupt.
1861 */
1862 static void
1863 ixgbe_update_link_status(struct adapter *adapter)
1864 {
1865         struct ifnet    *ifp = adapter->ifp;
1866         device_t dev = adapter->dev;
1867
1868
1869         if (adapter->link_up){
1870                 if (adapter->link_active == FALSE) {
1871                         if (bootverbose)
1872                                 device_printf(dev,"Link is up %d Gbps %s \n",
1873                                     ((adapter->link_speed == 128)? 10:1),
1874                                     "Full Duplex");
1875                         adapter->link_active = TRUE;
1876                         /* Update any Flow Control changes */
1877                         ixgbe_fc_enable(&adapter->hw);
1878                         if_link_state_change(ifp, LINK_STATE_UP);
1879                 }
1880         } else { /* Link down */
1881                 if (adapter->link_active == TRUE) {
1882                         if (bootverbose)
1883                                 device_printf(dev,"Link is Down\n");
1884                         if_link_state_change(ifp, LINK_STATE_DOWN);
1885                         adapter->link_active = FALSE;
1886                 }
1887         }
1888
1889         return;
1890 }
1891
1892
1893 /*********************************************************************
1894  *
1895  *  This routine disables all traffic on the adapter by issuing a
1896  *  global reset on the MAC and deallocates TX/RX buffers.
1897  *
1898  **********************************************************************/
1899
1900 static void
1901 ixgbe_stop(void *arg)
1902 {
1903         struct ifnet   *ifp;
1904         struct adapter *adapter = arg;
1905         struct ixgbe_hw *hw = &adapter->hw;
1906         ifp = adapter->ifp;
1907
1908         mtx_assert(&adapter->core_mtx, MA_OWNED);
1909
1910         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1911         ixgbe_disable_intr(adapter);
1912         callout_stop(&adapter->timer);
1913
1914         /* Let the stack know...*/
1915         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1916
1917         ixgbe_reset_hw(hw);
1918         hw->adapter_stopped = FALSE;
1919         ixgbe_stop_adapter(hw);
1920         if (hw->mac.type == ixgbe_mac_82599EB)
1921                 ixgbe_stop_mac_link_on_d3_82599(hw);
1922         /* Turn off the laser - noop with no optics */
1923         ixgbe_disable_tx_laser(hw);
1924
1925         /* Update the stack */
1926         adapter->link_up = FALSE;
1927         ixgbe_update_link_status(adapter);
1928
1929         /* reprogram the RAR[0] in case user changed it. */
1930         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1931
1932         return;
1933 }
1934
1935
1936 /*********************************************************************
1937  *
1938  *  Determine hardware revision.
1939  *
1940  **********************************************************************/
1941 static void
1942 ixgbe_identify_hardware(struct adapter *adapter)
1943 {
1944         device_t        dev = adapter->dev;
1945         struct ixgbe_hw *hw = &adapter->hw;
1946
1947         /* Save off the information about this board */
1948         hw->vendor_id = pci_get_vendor(dev);
1949         hw->device_id = pci_get_device(dev);
1950         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
1951         hw->subsystem_vendor_id =
1952             pci_read_config(dev, PCIR_SUBVEND_0, 2);
1953         hw->subsystem_device_id =
1954             pci_read_config(dev, PCIR_SUBDEV_0, 2);
1955
1956         /*
1957         ** Make sure BUSMASTER is set
1958         */
1959         pci_enable_busmaster(dev);
1960
1961         /* We need this here to set the num_segs below */
1962         ixgbe_set_mac_type(hw);
1963
1964         /* Pick up the 82599 and VF settings */
1965         if (hw->mac.type != ixgbe_mac_82598EB) {
1966                 hw->phy.smart_speed = ixgbe_smart_speed;
1967                 adapter->num_segs = IXGBE_82599_SCATTER;
1968         } else
1969                 adapter->num_segs = IXGBE_82598_SCATTER;
1970
1971         return;
1972 }
1973
1974 /*********************************************************************
1975  *
1976  *  Determine optic type
1977  *
1978  **********************************************************************/
1979 static void
1980 ixgbe_setup_optics(struct adapter *adapter)
1981 {
1982         struct ixgbe_hw *hw = &adapter->hw;
1983         int             layer;
1984
1985         layer = ixgbe_get_supported_physical_layer(hw);
1986
1987         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
1988                 adapter->optics = IFM_10G_T;
1989                 return;
1990         }
1991
1992         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
1993                 adapter->optics = IFM_1000_T;
1994                 return;
1995         }
1996
1997         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
1998                 adapter->optics = IFM_1000_SX;
1999                 return;
2000         }
2001
2002         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2003             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2004                 adapter->optics = IFM_10G_LR;
2005                 return;
2006         }
2007
2008         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2009                 adapter->optics = IFM_10G_SR;
2010                 return;
2011         }
2012
2013         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2014                 adapter->optics = IFM_10G_TWINAX;
2015                 return;
2016         }
2017
2018         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2019             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2020                 adapter->optics = IFM_10G_CX4;
2021                 return;
2022         }
2023
2024         /* If we get here just set the default */
2025         adapter->optics = IFM_ETHER | IFM_AUTO;
2026         return;
2027 }
2028
2029 /*********************************************************************
2030  *
2031  *  Setup the Legacy or MSI Interrupt handler
2032  *
2033  **********************************************************************/
2034 static int
2035 ixgbe_allocate_legacy(struct adapter *adapter)
2036 {
2037         device_t        dev = adapter->dev;
2038         struct          ix_queue *que = adapter->queues;
2039 #ifndef IXGBE_LEGACY_TX
2040         struct tx_ring          *txr = adapter->tx_rings;
2041 #endif
2042         int             error, rid = 0;
2043
2044         /* MSI RID at 1 */
2045         if (adapter->msix == 1)
2046                 rid = 1;
2047
2048         /* We allocate a single interrupt resource */
2049         adapter->res = bus_alloc_resource_any(dev,
2050             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2051         if (adapter->res == NULL) {
2052                 device_printf(dev, "Unable to allocate bus resource: "
2053                     "interrupt\n");
2054                 return (ENXIO);
2055         }
2056
2057         /*
2058          * Try allocating a fast interrupt and the associated deferred
2059          * processing contexts.
2060          */
2061 #ifndef IXGBE_LEGACY_TX
2062         TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2063 #endif
2064         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2065         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2066             taskqueue_thread_enqueue, &que->tq);
2067         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2068             device_get_nameunit(adapter->dev));
2069
2070         /* Tasklets for Link, SFP and Multispeed Fiber */
2071         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2072         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2073         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2074 #ifdef IXGBE_FDIR
2075         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2076 #endif
2077         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2078             taskqueue_thread_enqueue, &adapter->tq);
2079         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2080             device_get_nameunit(adapter->dev));
2081
2082         if ((error = bus_setup_intr(dev, adapter->res,
2083             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2084             que, &adapter->tag)) != 0) {
2085                 device_printf(dev, "Failed to register fast interrupt "
2086                     "handler: %d\n", error);
2087                 taskqueue_free(que->tq);
2088                 taskqueue_free(adapter->tq);
2089                 que->tq = NULL;
2090                 adapter->tq = NULL;
2091                 return (error);
2092         }
2093         /* For simplicity in the handlers */
2094         adapter->active_queues = IXGBE_EIMS_ENABLE_MASK;
2095
2096         return (0);
2097 }
2098
2099
2100 /*********************************************************************
2101  *
2102  *  Setup MSIX Interrupt resources and handlers
2103  *
2104  **********************************************************************/
2105 static int
2106 ixgbe_allocate_msix(struct adapter *adapter)
2107 {
2108         device_t        dev = adapter->dev;
2109         struct          ix_queue *que = adapter->queues;
2110         struct          tx_ring *txr = adapter->tx_rings;
2111         int             error, rid, vector = 0;
2112         int             cpu_id = 0;
2113 #ifdef  RSS
2114         cpuset_t        cpu_mask;
2115 #endif
2116
2117 #ifdef  RSS
2118         /*
2119          * If we're doing RSS, the number of queues needs to
2120          * match the number of RSS buckets that are configured.
2121          *
2122          * + If there's more queues than RSS buckets, we'll end
2123          *   up with queues that get no traffic.
2124          *
2125          * + If there's more RSS buckets than queues, we'll end
2126          *   up having multiple RSS buckets map to the same queue,
2127          *   so there'll be some contention.
2128          */
2129         if (adapter->num_queues != rss_getnumbuckets()) {
2130                 device_printf(dev,
2131                     "%s: number of queues (%d) != number of RSS buckets (%d)"
2132                     "; performance will be impacted.\n",
2133                     __func__,
2134                     adapter->num_queues,
2135                     rss_getnumbuckets());
2136         }
2137 #endif
2138
2139
2140
2141         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2142                 rid = vector + 1;
2143                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2144                     RF_SHAREABLE | RF_ACTIVE);
2145                 if (que->res == NULL) {
2146                         device_printf(dev,"Unable to allocate"
2147                             " bus resource: que interrupt [%d]\n", vector);
2148                         return (ENXIO);
2149                 }
2150                 /* Set the handler function */
2151                 error = bus_setup_intr(dev, que->res,
2152                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2153                     ixgbe_msix_que, que, &que->tag);
2154                 if (error) {
2155                         que->res = NULL;
2156                         device_printf(dev, "Failed to register QUE handler");
2157                         return (error);
2158                 }
2159 #if __FreeBSD_version >= 800504
2160                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2161 #endif
2162                 que->msix = vector;
2163                 adapter->active_queues |= (u64)(1 << que->msix);
2164 #ifdef  RSS
2165                 /*
2166                  * The queue ID is used as the RSS layer bucket ID.
2167                  * We look up the queue ID -> RSS CPU ID and select
2168                  * that.
2169                  */
2170                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
2171 #else
2172                 /*
2173                  * Bind the msix vector, and thus the
2174                  * rings to the corresponding cpu.
2175                  *
2176                  * This just happens to match the default RSS round-robin
2177                  * bucket -> queue -> CPU allocation.
2178                  */
2179                 if (adapter->num_queues > 1)
2180                         cpu_id = i;
2181 #endif
2182                 if (adapter->num_queues > 1)
2183                         bus_bind_intr(dev, que->res, cpu_id);
2184
2185 #ifdef  RSS
2186                 device_printf(dev,
2187                     "Bound RSS bucket %d to CPU %d\n",
2188                     i, cpu_id);
2189 #else
2190 #if 0 // This is too noisy
2191                 device_printf(dev,
2192                     "Bound queue %d to cpu %d\n",
2193                     i, cpu_id);
2194 #endif
2195 #endif
2196
2197
2198 #ifndef IXGBE_LEGACY_TX
2199                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2200 #endif
2201                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2202                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2203                     taskqueue_thread_enqueue, &que->tq);
2204 #ifdef  RSS
2205                 CPU_SETOF(cpu_id, &cpu_mask);
2206                 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2207                     &cpu_mask,
2208                     "%s (bucket %d)",
2209                     device_get_nameunit(adapter->dev),
2210                     cpu_id);
2211 #else
2212                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2213                     device_get_nameunit(adapter->dev));
2214 #endif
2215         }
2216
2217         /* and Link */
2218         rid = vector + 1;
2219         adapter->res = bus_alloc_resource_any(dev,
2220             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2221         if (!adapter->res) {
2222                 device_printf(dev,"Unable to allocate"
2223             " bus resource: Link interrupt [%d]\n", rid);
2224                 return (ENXIO);
2225         }
2226         /* Set the link handler function */
2227         error = bus_setup_intr(dev, adapter->res,
2228             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2229             ixgbe_msix_link, adapter, &adapter->tag);
2230         if (error) {
2231                 adapter->res = NULL;
2232                 device_printf(dev, "Failed to register LINK handler");
2233                 return (error);
2234         }
2235 #if __FreeBSD_version >= 800504
2236         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2237 #endif
2238         adapter->vector = vector;
2239         /* Tasklets for Link, SFP and Multispeed Fiber */
2240         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2241         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2242         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2243 #ifdef IXGBE_FDIR
2244         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2245 #endif
2246         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2247             taskqueue_thread_enqueue, &adapter->tq);
2248         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2249             device_get_nameunit(adapter->dev));
2250
2251         return (0);
2252 }
2253
2254 /*
2255  * Setup Either MSI/X or MSI
2256  */
2257 static int
2258 ixgbe_setup_msix(struct adapter *adapter)
2259 {
2260         device_t dev = adapter->dev;
2261         int rid, want, queues, msgs;
2262
2263         /* Override by tuneable */
2264         if (ixgbe_enable_msix == 0)
2265                 goto msi;
2266
2267         /* First try MSI/X */
2268         msgs = pci_msix_count(dev);
2269         if (msgs == 0)
2270                 goto msi;
2271         rid = PCIR_BAR(MSIX_82598_BAR);
2272         adapter->msix_mem = bus_alloc_resource_any(dev,
2273             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2274         if (adapter->msix_mem == NULL) {
2275                 rid += 4;       /* 82599 maps in higher BAR */
2276                 adapter->msix_mem = bus_alloc_resource_any(dev,
2277                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2278         }
2279         if (adapter->msix_mem == NULL) {
2280                 /* May not be enabled */
2281                 device_printf(adapter->dev,
2282                     "Unable to map MSIX table \n");
2283                 goto msi;
2284         }
2285
2286         /* Figure out a reasonable auto config value */
2287         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2288
2289 #ifdef  RSS
2290         /* If we're doing RSS, clamp at the number of RSS buckets */
2291         if (queues > rss_getnumbuckets())
2292                 queues = rss_getnumbuckets();
2293 #endif
2294
2295         if (ixgbe_num_queues != 0)
2296                 queues = ixgbe_num_queues;
2297
2298         /* reflect correct sysctl value */
2299         ixgbe_num_queues = queues;
2300
2301         /*
2302         ** Want one vector (RX/TX pair) per queue
2303         ** plus an additional for Link.
2304         */
2305         want = queues + 1;
2306         if (msgs >= want)
2307                 msgs = want;
2308         else {
2309                 device_printf(adapter->dev,
2310                     "MSIX Configuration Problem, "
2311                     "%d vectors but %d queues wanted!\n",
2312                     msgs, want);
2313                 goto msi;
2314         }
2315         if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2316                 device_printf(adapter->dev,
2317                     "Using MSIX interrupts with %d vectors\n", msgs);
2318                 adapter->num_queues = queues;
2319                 return (msgs);
2320         }
2321         /*
2322         ** If MSIX alloc failed or provided us with
2323         ** less than needed, free and fall through to MSI
2324         */
2325         pci_release_msi(dev);
2326
2327 msi:
2328         if (adapter->msix_mem != NULL) {
2329                 bus_release_resource(dev, SYS_RES_MEMORY,
2330                     rid, adapter->msix_mem);
2331                 adapter->msix_mem = NULL;
2332         }
2333         msgs = 1;
2334         if (pci_alloc_msi(dev, &msgs) == 0) {
2335                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2336                 return (msgs);
2337         }
2338         device_printf(adapter->dev,"Using a Legacy interrupt\n");
2339         return (0);
2340 }
2341
2342
2343 static int
2344 ixgbe_allocate_pci_resources(struct adapter *adapter)
2345 {
2346         int             rid;
2347         device_t        dev = adapter->dev;
2348
2349         rid = PCIR_BAR(0);
2350         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2351             &rid, RF_ACTIVE);
2352
2353         if (!(adapter->pci_mem)) {
2354                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2355                 return (ENXIO);
2356         }
2357
2358         adapter->osdep.mem_bus_space_tag =
2359                 rman_get_bustag(adapter->pci_mem);
2360         adapter->osdep.mem_bus_space_handle =
2361                 rman_get_bushandle(adapter->pci_mem);
2362         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2363
2364         /* Legacy defaults */
2365         adapter->num_queues = 1;
2366         adapter->hw.back = &adapter->osdep;
2367
2368         /*
2369         ** Now setup MSI or MSI/X, should
2370         ** return us the number of supported
2371         ** vectors. (Will be 1 for MSI)
2372         */
2373         adapter->msix = ixgbe_setup_msix(adapter);
2374         return (0);
2375 }
2376
2377 static void
2378 ixgbe_free_pci_resources(struct adapter * adapter)
2379 {
2380         struct          ix_queue *que = adapter->queues;
2381         device_t        dev = adapter->dev;
2382         int             rid, memrid;
2383
2384         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2385                 memrid = PCIR_BAR(MSIX_82598_BAR);
2386         else
2387                 memrid = PCIR_BAR(MSIX_82599_BAR);
2388
2389         /*
2390         ** There is a slight possibility of a failure mode
2391         ** in attach that will result in entering this function
2392         ** before interrupt resources have been initialized, and
2393         ** in that case we do not want to execute the loops below
2394         ** We can detect this reliably by the state of the adapter
2395         ** res pointer.
2396         */
2397         if (adapter->res == NULL)
2398                 goto mem;
2399
2400         /*
2401         **  Release all msix queue resources:
2402         */
2403         for (int i = 0; i < adapter->num_queues; i++, que++) {
2404                 rid = que->msix + 1;
2405                 if (que->tag != NULL) {
2406                         bus_teardown_intr(dev, que->res, que->tag);
2407                         que->tag = NULL;
2408                 }
2409                 if (que->res != NULL)
2410                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2411         }
2412
2413
2414         /* Clean the Legacy or Link interrupt last */
2415         if (adapter->vector) /* we are doing MSIX */
2416                 rid = adapter->vector + 1;
2417         else
2418                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2419
2420         if (adapter->tag != NULL) {
2421                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2422                 adapter->tag = NULL;
2423         }
2424         if (adapter->res != NULL)
2425                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2426
2427 mem:
2428         if (adapter->msix)
2429                 pci_release_msi(dev);
2430
2431         if (adapter->msix_mem != NULL)
2432                 bus_release_resource(dev, SYS_RES_MEMORY,
2433                     memrid, adapter->msix_mem);
2434
2435         if (adapter->pci_mem != NULL)
2436                 bus_release_resource(dev, SYS_RES_MEMORY,
2437                     PCIR_BAR(0), adapter->pci_mem);
2438
2439         return;
2440 }
2441
2442 /*********************************************************************
2443  *
2444  *  Setup networking device structure and register an interface.
2445  *
2446  **********************************************************************/
2447 static int
2448 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2449 {
2450         struct ifnet   *ifp;
2451
2452         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2453
2454         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2455         if (ifp == NULL) {
2456                 device_printf(dev, "can not allocate ifnet structure\n");
2457                 return (-1);
2458         }
2459         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2460         ifp->if_baudrate = IF_Gbps(10);
2461         ifp->if_init = ixgbe_init;
2462         ifp->if_softc = adapter;
2463         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2464         ifp->if_ioctl = ixgbe_ioctl;
2465 #if __FreeBSD_version >= 1100036
2466         if_setgetcounterfn(ifp, ixgbe_get_counter);
2467 #endif
2468 #ifndef IXGBE_LEGACY_TX
2469         ifp->if_transmit = ixgbe_mq_start;
2470         ifp->if_qflush = ixgbe_qflush;
2471 #else
2472         ifp->if_start = ixgbe_start;
2473         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2474         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2475         IFQ_SET_READY(&ifp->if_snd);
2476 #endif
2477
2478         ether_ifattach(ifp, adapter->hw.mac.addr);
2479
2480         adapter->max_frame_size =
2481             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2482
2483         /*
2484          * Tell the upper layer(s) we support long frames.
2485          */
2486         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2487
2488         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2489         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2490         ifp->if_capabilities |= IFCAP_LRO;
2491         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2492                              |  IFCAP_VLAN_HWTSO
2493                              |  IFCAP_VLAN_MTU
2494                              |  IFCAP_HWSTATS;
2495         ifp->if_capenable = ifp->if_capabilities;
2496
2497         /*
2498         ** Don't turn this on by default, if vlans are
2499         ** created on another pseudo device (eg. lagg)
2500         ** then vlan events are not passed thru, breaking
2501         ** operation, but with HW FILTER off it works. If
2502         ** using vlans directly on the ixgbe driver you can
2503         ** enable this and get full hardware tag filtering.
2504         */
2505         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2506
2507         /*
2508          * Specify the media types supported by this adapter and register
2509          * callbacks to update media and link information
2510          */
2511         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2512                     ixgbe_media_status);
2513
2514         ixgbe_add_media_types(adapter);
2515
2516         /* Autoselect media by default */
2517         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2518
2519         return (0);
2520 }
2521
2522 static void
2523 ixgbe_add_media_types(struct adapter *adapter)
2524 {
2525         struct ixgbe_hw *hw = &adapter->hw;
2526         device_t dev = adapter->dev;
2527         int layer;
2528
2529         layer = ixgbe_get_supported_physical_layer(hw);
2530
2531         /* Media types with matching FreeBSD media defines */
2532         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T)
2533                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2534         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T)
2535                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2536         if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
2537                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2538
2539         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
2540             layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
2541                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2542
2543         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
2544                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2545         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR)
2546                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2547         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
2548                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2549         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
2550                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2551 #if 0
2552         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_LX)
2553                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2554 #endif
2555
2556         /*
2557         ** Other (no matching FreeBSD media type):
2558         ** To workaround this, we'll assign these completely
2559         ** inappropriate media types.
2560         */
2561         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
2562                 device_printf(dev, "Media supported: 10GbaseKR\n");
2563                 device_printf(dev, "10GbaseKR mapped to 10baseT\n");
2564                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2565         }
2566         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) {
2567                 device_printf(dev, "Media supported: 10GbaseKX4\n");
2568                 device_printf(dev, "10GbaseKX4 mapped to 10base2\n");
2569                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_2, 0, NULL);
2570         }
2571         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) {
2572                 device_printf(dev, "Media supported: 1000baseKX\n");
2573                 device_printf(dev, "1000baseKX mapped to 10base5\n");
2574                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_5, 0, NULL);
2575         }
2576         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) {
2577                 /* Someday, someone will care about you... */
2578                 device_printf(dev, "Media supported: 1000baseBX\n");
2579         }
2580
2581         /* Very old */
2582         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2583                 ifmedia_add(&adapter->media,
2584                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2585                 ifmedia_add(&adapter->media,
2586                     IFM_ETHER | IFM_1000_T, 0, NULL);
2587         }
2588
2589         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2590 }
2591
2592 static void
2593 ixgbe_config_link(struct adapter *adapter)
2594 {
2595         struct ixgbe_hw *hw = &adapter->hw;
2596         u32     autoneg, err = 0;
2597         bool    sfp, negotiate;
2598
2599         sfp = ixgbe_is_sfp(hw);
2600
2601         if (sfp) {
2602                 if (hw->phy.multispeed_fiber) {
2603                         hw->mac.ops.setup_sfp(hw);
2604                         ixgbe_enable_tx_laser(hw);
2605                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2606                 } else
2607                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2608         } else {
2609                 if (hw->mac.ops.check_link)
2610                         err = ixgbe_check_link(hw, &adapter->link_speed,
2611                             &adapter->link_up, FALSE);
2612                 if (err)
2613                         goto out;
2614                 autoneg = hw->phy.autoneg_advertised;
2615                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2616                         err  = hw->mac.ops.get_link_capabilities(hw,
2617                             &autoneg, &negotiate);
2618                 if (err)
2619                         goto out;
2620                 if (hw->mac.ops.setup_link)
2621                         err = hw->mac.ops.setup_link(hw,
2622                             autoneg, adapter->link_up);
2623         }
2624 out:
2625         return;
2626 }
2627
2628
2629 /*********************************************************************
2630  *
2631  *  Enable transmit units.
2632  *
2633  **********************************************************************/
2634 static void
2635 ixgbe_initialize_transmit_units(struct adapter *adapter)
2636 {
2637         struct tx_ring  *txr = adapter->tx_rings;
2638         struct ixgbe_hw *hw = &adapter->hw;
2639
2640         /* Setup the Base and Length of the Tx Descriptor Ring */
2641
2642         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2643                 u64     tdba = txr->txdma.dma_paddr;
2644                 u32     txctrl = 0;
2645
2646                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2647                        (tdba & 0x00000000ffffffffULL));
2648                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2649                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2650                     adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
2651
2652                 /* Setup the HW Tx Head and Tail descriptor pointers */
2653                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2654                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2655
2656                 /* Cache the tail address */
2657                 txr->tail = IXGBE_TDT(txr->me);
2658
2659                 /* Set the processing limit */
2660                 txr->process_limit = ixgbe_tx_process_limit;
2661
2662                 /* Disable Head Writeback */
2663                 switch (hw->mac.type) {
2664                 case ixgbe_mac_82598EB:
2665                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2666                         break;
2667                 case ixgbe_mac_82599EB:
2668                 case ixgbe_mac_X540:
2669                 default:
2670                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2671                         break;
2672                 }
2673                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2674                 switch (hw->mac.type) {
2675                 case ixgbe_mac_82598EB:
2676                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2677                         break;
2678                 case ixgbe_mac_82599EB:
2679                 case ixgbe_mac_X540:
2680                 default:
2681                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2682                         break;
2683                 }
2684
2685         }
2686
2687         if (hw->mac.type != ixgbe_mac_82598EB) {
2688                 u32 dmatxctl, rttdcs;
2689                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2690                 dmatxctl |= IXGBE_DMATXCTL_TE;
2691                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2692                 /* Disable arbiter to set MTQC */
2693                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2694                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2695                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2696                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2697                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2698                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2699         }
2700
2701         return;
2702 }
2703
2704 static void
2705 ixgbe_initialise_rss_mapping(struct adapter *adapter)
2706 {
2707         struct ixgbe_hw *hw = &adapter->hw;
2708         uint32_t reta;
2709         int i, j, queue_id;
2710         uint32_t rss_key[10];
2711         uint32_t mrqc;
2712 #ifdef  RSS
2713         uint32_t rss_hash_config;
2714 #endif
2715
2716         /* Setup RSS */
2717         reta = 0;
2718
2719 #ifdef  RSS
2720         /* Fetch the configured RSS key */
2721         rss_getkey((uint8_t *) &rss_key);
2722 #else
2723         /* set up random bits */
2724         arc4rand(&rss_key, sizeof(rss_key), 0);
2725 #endif
2726
2727         /* Set up the redirection table */
2728         for (i = 0, j = 0; i < 128; i++, j++) {
2729                 if (j == adapter->num_queues) j = 0;
2730 #ifdef  RSS
2731                 /*
2732                  * Fetch the RSS bucket id for the given indirection entry.
2733                  * Cap it at the number of configured buckets (which is
2734                  * num_queues.)
2735                  */
2736                 queue_id = rss_get_indirection_to_bucket(i);
2737                 queue_id = queue_id % adapter->num_queues;
2738 #else
2739                 queue_id = (j * 0x11);
2740 #endif
2741                 /*
2742                  * The low 8 bits are for hash value (n+0);
2743                  * The next 8 bits are for hash value (n+1), etc.
2744                  */
2745                 reta = reta >> 8;
2746                 reta = reta | ( ((uint32_t) queue_id) << 24);
2747                 if ((i & 3) == 3) {
2748                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2749                         reta = 0;
2750                 }
2751         }
2752
2753         /* Now fill our hash function seeds */
2754         for (int i = 0; i < 10; i++)
2755                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
2756
2757         /* Perform hash on these packet types */
2758 #ifdef  RSS
2759         mrqc = IXGBE_MRQC_RSSEN;
2760         rss_hash_config = rss_gethashconfig();
2761         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
2762                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
2763         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
2764                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
2765         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
2766                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
2767         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
2768                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2769         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
2770                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
2771         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
2772                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
2773         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
2774                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
2775         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4_EX)
2776                 device_printf(adapter->dev,
2777                     "%s: RSS_HASHTYPE_RSS_UDP_IPV4_EX defined, "
2778                     "but not supported\n", __func__);
2779         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
2780                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
2781         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
2782                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2783 #else
2784         /*
2785          * Disable UDP - IP fragments aren't currently being handled
2786          * and so we end up with a mix of 2-tuple and 4-tuple
2787          * traffic.
2788          */
2789         mrqc = IXGBE_MRQC_RSSEN
2790              | IXGBE_MRQC_RSS_FIELD_IPV4
2791              | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2792 #if 0
2793              | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
2794 #endif
2795              | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
2796              | IXGBE_MRQC_RSS_FIELD_IPV6_EX
2797              | IXGBE_MRQC_RSS_FIELD_IPV6
2798              | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
2799 #if 0
2800              | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
2801              | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP
2802 #endif
2803         ;
2804 #endif /* RSS */
2805         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2806 }
2807
2808
2809 /*********************************************************************
2810  *
2811  *  Setup receive registers and features.
2812  *
2813  **********************************************************************/
2814 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2815
2816 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
2817
2818 static void
2819 ixgbe_initialize_receive_units(struct adapter *adapter)
2820 {
2821         struct  rx_ring *rxr = adapter->rx_rings;
2822         struct ixgbe_hw *hw = &adapter->hw;
2823         struct ifnet   *ifp = adapter->ifp;
2824         u32             bufsz, fctrl, srrctl, rxcsum;
2825         u32             hlreg;
2826
2827
2828         /*
2829          * Make sure receives are disabled while
2830          * setting up the descriptor ring
2831          */
2832         ixgbe_disable_rx(hw);
2833
2834         /* Enable broadcasts */
2835         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2836         fctrl |= IXGBE_FCTRL_BAM;
2837         fctrl |= IXGBE_FCTRL_DPF;
2838         fctrl |= IXGBE_FCTRL_PMCF;
2839         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2840
2841         /* Set for Jumbo Frames? */
2842         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2843         if (ifp->if_mtu > ETHERMTU)
2844                 hlreg |= IXGBE_HLREG0_JUMBOEN;
2845         else
2846                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
2847 #ifdef DEV_NETMAP
2848         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
2849         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
2850                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
2851         else
2852                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
2853 #endif /* DEV_NETMAP */
2854         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
2855
2856         bufsz = (adapter->rx_mbuf_sz +
2857             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2858
2859         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
2860                 u64 rdba = rxr->rxdma.dma_paddr;
2861
2862                 /* Setup the Base and Length of the Rx Descriptor Ring */
2863                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
2864                                (rdba & 0x00000000ffffffffULL));
2865                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
2866                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
2867                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
2868
2869                 /* Set up the SRRCTL register */
2870                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
2871                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2872                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2873                 srrctl |= bufsz;
2874                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2875
2876                 /*
2877                  * Set DROP_EN iff we have no flow control and >1 queue.
2878                  * Note that srrctl was cleared shortly before during reset,
2879                  * so we do not need to clear the bit, but do it just in case
2880                  * this code is moved elsewhere.
2881                  */
2882                 if (adapter->num_queues > 1 &&
2883                     adapter->hw.fc.requested_mode == ixgbe_fc_none) {
2884                         srrctl |= IXGBE_SRRCTL_DROP_EN;
2885                 } else {
2886                         srrctl &= ~IXGBE_SRRCTL_DROP_EN;
2887                 }
2888
2889                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
2890
2891                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
2892                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
2893                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
2894
2895                 /* Set the processing limit */
2896                 rxr->process_limit = ixgbe_rx_process_limit;
2897
2898                 /* Set the driver rx tail address */
2899                 rxr->tail =  IXGBE_RDT(rxr->me);
2900         }
2901
2902         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
2903                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
2904                               IXGBE_PSRTYPE_UDPHDR |
2905                               IXGBE_PSRTYPE_IPV4HDR |
2906                               IXGBE_PSRTYPE_IPV6HDR;
2907                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
2908         }
2909
2910         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2911
2912         ixgbe_initialise_rss_mapping(adapter);
2913
2914         if (adapter->num_queues > 1) {
2915                 /* RSS and RX IPP Checksum are mutually exclusive */
2916                 rxcsum |= IXGBE_RXCSUM_PCSD;
2917         }
2918
2919         if (ifp->if_capenable & IFCAP_RXCSUM)
2920                 rxcsum |= IXGBE_RXCSUM_PCSD;
2921
2922         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
2923                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
2924
2925         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2926
2927         return;
2928 }
2929
2930
2931 /*
2932 ** This routine is run via an vlan config EVENT,
2933 ** it enables us to use the HW Filter table since
2934 ** we can get the vlan id. This just creates the
2935 ** entry in the soft version of the VFTA, init will
2936 ** repopulate the real table.
2937 */
2938 static void
2939 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
2940 {
2941         struct adapter  *adapter = ifp->if_softc;
2942         u16             index, bit;
2943
2944         if (ifp->if_softc !=  arg)   /* Not our event */
2945                 return;
2946
2947         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
2948                 return;
2949
2950         IXGBE_CORE_LOCK(adapter);
2951         index = (vtag >> 5) & 0x7F;
2952         bit = vtag & 0x1F;
2953         adapter->shadow_vfta[index] |= (1 << bit);
2954         ++adapter->num_vlans;
2955         ixgbe_setup_vlan_hw_support(adapter);
2956         IXGBE_CORE_UNLOCK(adapter);
2957 }
2958
2959 /*
2960 ** This routine is run via an vlan
2961 ** unconfig EVENT, remove our entry
2962 ** in the soft vfta.
2963 */
2964 static void
2965 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
2966 {
2967         struct adapter  *adapter = ifp->if_softc;
2968         u16             index, bit;
2969
2970         if (ifp->if_softc !=  arg)
2971                 return;
2972
2973         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
2974                 return;
2975
2976         IXGBE_CORE_LOCK(adapter);
2977         index = (vtag >> 5) & 0x7F;
2978         bit = vtag & 0x1F;
2979         adapter->shadow_vfta[index] &= ~(1 << bit);
2980         --adapter->num_vlans;
2981         /* Re-init to load the changes */
2982         ixgbe_setup_vlan_hw_support(adapter);
2983         IXGBE_CORE_UNLOCK(adapter);
2984 }
2985
2986 static void
2987 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
2988 {
2989         struct ifnet    *ifp = adapter->ifp;
2990         struct ixgbe_hw *hw = &adapter->hw;
2991         struct rx_ring  *rxr;
2992         u32             ctrl;
2993
2994
2995         /*
2996         ** We get here thru init_locked, meaning
2997         ** a soft reset, this has already cleared
2998         ** the VFTA and other state, so if there
2999         ** have been no vlan's registered do nothing.
3000         */
3001         if (adapter->num_vlans == 0)
3002                 return;
3003
3004         /* Setup the queues for vlans */
3005         for (int i = 0; i < adapter->num_queues; i++) {
3006                 rxr = &adapter->rx_rings[i];
3007                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
3008                 if (hw->mac.type != ixgbe_mac_82598EB) {
3009                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
3010                         ctrl |= IXGBE_RXDCTL_VME;
3011                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
3012                 }
3013                 rxr->vtag_strip = TRUE;
3014         }
3015
3016         if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
3017                 return;
3018         /*
3019         ** A soft reset zero's out the VFTA, so
3020         ** we need to repopulate it now.
3021         */
3022         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
3023                 if (adapter->shadow_vfta[i] != 0)
3024                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
3025                             adapter->shadow_vfta[i]);
3026
3027         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3028         /* Enable the Filter Table if enabled */
3029         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3030                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
3031                 ctrl |= IXGBE_VLNCTRL_VFE;
3032         }
3033         if (hw->mac.type == ixgbe_mac_82598EB)
3034                 ctrl |= IXGBE_VLNCTRL_VME;
3035         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
3036 }
3037
3038 static void
3039 ixgbe_enable_intr(struct adapter *adapter)
3040 {
3041         struct ixgbe_hw *hw = &adapter->hw;
3042         struct ix_queue *que = adapter->queues;
3043         u32             mask, fwsm;
3044
3045         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3046         /* Enable Fan Failure detection */
3047         if (hw->device_id == IXGBE_DEV_ID_82598AT)
3048                     mask |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw);
3049
3050         switch (adapter->hw.mac.type) {
3051                 case ixgbe_mac_82599EB:
3052                         mask |= IXGBE_EIMS_ECC;
3053                         /* Temperature sensor on some adapters */
3054                         mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
3055                         /* SFP+ (RX_LOS_N & MOD_ABS_N) */
3056                         mask |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw);
3057                         mask |= IXGBE_EIMS_GPI_SDP2_BY_MAC(hw);
3058 #ifdef IXGBE_FDIR
3059                         mask |= IXGBE_EIMS_FLOW_DIR;
3060 #endif
3061                         break;
3062                 case ixgbe_mac_X540:
3063                 case ixgbe_mac_X550:
3064                 case ixgbe_mac_X550EM_a:
3065                 case ixgbe_mac_X550EM_x:
3066                         /* Detect if Thermal Sensor is enabled */
3067                         fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
3068                         if (fwsm & IXGBE_FWSM_TS_ENABLED)
3069                                 mask |= IXGBE_EIMS_TS;
3070                         /* XXX: Which SFP mode line does this look at? */
3071                         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP)
3072                                 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
3073                         mask |= IXGBE_EIMS_ECC;
3074 #ifdef IXGBE_FDIR
3075                         mask |= IXGBE_EIMS_FLOW_DIR;
3076 #endif
3077                 /* falls through */
3078                 default:
3079                         break;
3080         }
3081
3082         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3083
3084         /* With RSS we use auto clear */
3085         if (adapter->msix_mem) {
3086                 mask = IXGBE_EIMS_ENABLE_MASK;
3087                 /* Don't autoclear Link */
3088                 mask &= ~IXGBE_EIMS_OTHER;
3089                 mask &= ~IXGBE_EIMS_LSC;
3090                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
3091         }
3092
3093         /*
3094         ** Now enable all queues, this is done separately to
3095         ** allow for handling the extended (beyond 32) MSIX
3096         ** vectors that can be used by 82599
3097         */
3098         for (int i = 0; i < adapter->num_queues; i++, que++)
3099                 ixgbe_enable_queue(adapter, que->msix);
3100
3101         IXGBE_WRITE_FLUSH(hw);
3102
3103         return;
3104 }
3105
3106 static void
3107 ixgbe_disable_intr(struct adapter *adapter)
3108 {
3109         if (adapter->msix_mem)
3110                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
3111         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
3112                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
3113         } else {
3114                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
3115                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
3116                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
3117         }
3118         IXGBE_WRITE_FLUSH(&adapter->hw);
3119         return;
3120 }
3121
3122 /*
3123 ** Get the width and transaction speed of
3124 ** the slot this adapter is plugged into.
3125 */
3126 static void
3127 ixgbe_get_slot_info(struct ixgbe_hw *hw)
3128 {
3129         device_t                dev = ((struct ixgbe_osdep *)hw->back)->dev;
3130         struct ixgbe_mac_info   *mac = &hw->mac;
3131         u16                     link;
3132         u32                     offset;
3133
3134         /* For most devices simply call the shared code routine */
3135         if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
3136                 ixgbe_get_bus_info(hw);
3137                 /* These devices don't use PCI-E */
3138                 if (hw->mac.type == ixgbe_mac_X550EM_x
3139                     || hw->mac.type == ixgbe_mac_X550EM_a)
3140                         return;
3141                 goto display;
3142         }
3143
3144         /*
3145         ** For the Quad port adapter we need to parse back
3146         ** up the PCI tree to find the speed of the expansion
3147         ** slot into which this adapter is plugged. A bit more work.
3148         */
3149         dev = device_get_parent(device_get_parent(dev));
3150 #ifdef IXGBE_DEBUG
3151         device_printf(dev, "parent pcib = %x,%x,%x\n",
3152             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
3153 #endif
3154         dev = device_get_parent(device_get_parent(dev));
3155 #ifdef IXGBE_DEBUG
3156         device_printf(dev, "slot pcib = %x,%x,%x\n",
3157             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
3158 #endif
3159         /* Now get the PCI Express Capabilities offset */
3160         pci_find_cap(dev, PCIY_EXPRESS, &offset);
3161         /* ...and read the Link Status Register */
3162         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
3163         switch (link & IXGBE_PCI_LINK_WIDTH) {
3164         case IXGBE_PCI_LINK_WIDTH_1:
3165                 hw->bus.width = ixgbe_bus_width_pcie_x1;
3166                 break;
3167         case IXGBE_PCI_LINK_WIDTH_2:
3168                 hw->bus.width = ixgbe_bus_width_pcie_x2;
3169                 break;
3170         case IXGBE_PCI_LINK_WIDTH_4:
3171                 hw->bus.width = ixgbe_bus_width_pcie_x4;
3172                 break;
3173         case IXGBE_PCI_LINK_WIDTH_8:
3174                 hw->bus.width = ixgbe_bus_width_pcie_x8;
3175                 break;
3176         default:
3177                 hw->bus.width = ixgbe_bus_width_unknown;
3178                 break;
3179         }
3180
3181         switch (link & IXGBE_PCI_LINK_SPEED) {
3182         case IXGBE_PCI_LINK_SPEED_2500:
3183                 hw->bus.speed = ixgbe_bus_speed_2500;
3184                 break;
3185         case IXGBE_PCI_LINK_SPEED_5000:
3186                 hw->bus.speed = ixgbe_bus_speed_5000;
3187                 break;
3188         case IXGBE_PCI_LINK_SPEED_8000:
3189                 hw->bus.speed = ixgbe_bus_speed_8000;
3190                 break;
3191         default:
3192                 hw->bus.speed = ixgbe_bus_speed_unknown;
3193                 break;
3194         }
3195
3196         mac->ops.set_lan_id(hw);
3197
3198 display:
3199         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
3200             ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
3201             (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
3202             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
3203             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
3204             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
3205             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
3206             ("Unknown"));
3207
3208         if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
3209             ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
3210             (hw->bus.speed == ixgbe_bus_speed_2500))) {
3211                 device_printf(dev, "PCI-Express bandwidth available"
3212                     " for this card\n     is not sufficient for"
3213                     " optimal performance.\n");
3214                 device_printf(dev, "For optimal performance a x8 "
3215                     "PCIE, or x4 PCIE Gen2 slot is required.\n");
3216         }
3217         if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
3218             ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
3219             (hw->bus.speed < ixgbe_bus_speed_8000))) {
3220                 device_printf(dev, "PCI-Express bandwidth available"
3221                     " for this card\n     is not sufficient for"
3222                     " optimal performance.\n");
3223                 device_printf(dev, "For optimal performance a x8 "
3224                     "PCIE Gen3 slot is required.\n");
3225         }
3226
3227         return;
3228 }
3229
3230
3231 /*
3232 ** Setup the correct IVAR register for a particular MSIX interrupt
3233 **   (yes this is all very magic and confusing :)
3234 **  - entry is the register array entry
3235 **  - vector is the MSIX vector for this queue
3236 **  - type is RX/TX/MISC
3237 */
3238 static void
3239 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
3240 {
3241         struct ixgbe_hw *hw = &adapter->hw;
3242         u32 ivar, index;
3243
3244         vector |= IXGBE_IVAR_ALLOC_VAL;
3245
3246         switch (hw->mac.type) {
3247
3248         case ixgbe_mac_82598EB:
3249                 if (type == -1)
3250                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
3251                 else
3252                         entry += (type * 64);
3253                 index = (entry >> 2) & 0x1F;
3254                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3255                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
3256                 ivar |= (vector << (8 * (entry & 0x3)));
3257                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
3258                 break;
3259
3260         case ixgbe_mac_82599EB:
3261         case ixgbe_mac_X540:
3262         case ixgbe_mac_X550:
3263         case ixgbe_mac_X550EM_a:
3264         case ixgbe_mac_X550EM_x:
3265                 if (type == -1) { /* MISC IVAR */
3266                         index = (entry & 1) * 8;
3267                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
3268                         ivar &= ~(0xFF << index);
3269                         ivar |= (vector << index);
3270                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
3271                 } else {        /* RX/TX IVARS */
3272                         index = (16 * (entry & 1)) + (8 * type);
3273                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
3274                         ivar &= ~(0xFF << index);
3275                         ivar |= (vector << index);
3276                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
3277                 }
3278
3279         default:
3280                 break;
3281         }
3282 }
3283
3284 static void
3285 ixgbe_configure_ivars(struct adapter *adapter)
3286 {
3287         struct  ix_queue *que = adapter->queues;
3288         u32 newitr;
3289
3290         if (ixgbe_max_interrupt_rate > 0)
3291                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
3292         else
3293                 newitr = 0;
3294
3295         for (int i = 0; i < adapter->num_queues; i++, que++) {
3296                 /* First the RX queue entry */
3297                 ixgbe_set_ivar(adapter, i, que->msix, 0);
3298                 /* ... and the TX */
3299                 ixgbe_set_ivar(adapter, i, que->msix, 1);
3300                 /* Set an Initial EITR value */
3301                 IXGBE_WRITE_REG(&adapter->hw,
3302                     IXGBE_EITR(que->msix), newitr);
3303         }
3304
3305         /* For the Link interrupt */
3306         ixgbe_set_ivar(adapter, 1, adapter->vector, -1);
3307 }
3308
3309 /*
3310 ** ixgbe_sfp_probe - called in the local timer to
3311 ** determine if a port had optics inserted.
3312 */
3313 static bool ixgbe_sfp_probe(struct adapter *adapter)
3314 {
3315         struct ixgbe_hw *hw = &adapter->hw;
3316         device_t        dev = adapter->dev;
3317         bool            result = FALSE;
3318
3319         if ((hw->phy.type == ixgbe_phy_nl) &&
3320             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
3321                 s32 ret = hw->phy.ops.identify_sfp(hw);
3322                 if (ret)
3323                         goto out;
3324                 ret = hw->phy.ops.reset(hw);
3325                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3326                         device_printf(dev,"Unsupported SFP+ module detected!");
3327                         printf(" Reload driver with supported module.\n");
3328                         adapter->sfp_probe = FALSE;
3329                         goto out;
3330                 } else
3331                         device_printf(dev,"SFP+ module detected!\n");
3332                 /* We now have supported optics */
3333                 adapter->sfp_probe = FALSE;
3334                 /* Set the optics type so system reports correctly */
3335                 ixgbe_setup_optics(adapter);
3336                 result = TRUE;
3337         }
3338 out:
3339         return (result);
3340 }
3341
3342 /*
3343 ** Tasklet handler for MSIX Link interrupts
3344 **  - do outside interrupt since it might sleep
3345 */
3346 static void
3347 ixgbe_handle_link(void *context, int pending)
3348 {
3349         struct adapter  *adapter = context;
3350
3351         ixgbe_check_link(&adapter->hw,
3352             &adapter->link_speed, &adapter->link_up, 0);
3353         ixgbe_update_link_status(adapter);
3354 }
3355
3356 /*
3357 ** Tasklet for handling SFP module interrupts
3358 */
3359 static void
3360 ixgbe_handle_mod(void *context, int pending)
3361 {
3362         struct adapter  *adapter = context;
3363         struct ixgbe_hw *hw = &adapter->hw;
3364         device_t        dev = adapter->dev;
3365         u32 err;
3366
3367         err = hw->phy.ops.identify_sfp(hw);
3368         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3369                 device_printf(dev,
3370                     "Unsupported SFP+ module type was detected.\n");
3371                 return;
3372         }
3373         err = hw->mac.ops.setup_sfp(hw);
3374         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3375                 device_printf(dev,
3376                     "Setup failure - unsupported SFP+ module type.\n");
3377                 return;
3378         }
3379         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
3380         return;
3381 }
3382
3383
3384 /*
3385 ** Tasklet for handling MSF (multispeed fiber) interrupts
3386 */
3387 static void
3388 ixgbe_handle_msf(void *context, int pending)
3389 {
3390         struct adapter  *adapter = context;
3391         struct ixgbe_hw *hw = &adapter->hw;
3392         u32 autoneg;
3393         bool negotiate;
3394         int err;
3395
3396         err = hw->phy.ops.identify_sfp(hw);
3397         if (!err) {
3398                 ixgbe_setup_optics(adapter);
3399                 INIT_DEBUGOUT1("ixgbe_sfp_probe: flags: %X\n", adapter->optics);
3400         }
3401
3402         autoneg = hw->phy.autoneg_advertised;
3403         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3404                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
3405         if (hw->mac.ops.setup_link)
3406                 hw->mac.ops.setup_link(hw, autoneg, TRUE);
3407
3408         ifmedia_removeall(&adapter->media);
3409         ixgbe_add_media_types(adapter);
3410         return;
3411 }
3412
3413 #ifdef IXGBE_FDIR
3414 /*
3415 ** Tasklet for reinitializing the Flow Director filter table
3416 */
3417 static void
3418 ixgbe_reinit_fdir(void *context, int pending)
3419 {
3420         struct adapter  *adapter = context;
3421         struct ifnet   *ifp = adapter->ifp;
3422
3423         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
3424                 return;
3425         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
3426         adapter->fdir_reinit = 0;
3427         /* re-enable flow director interrupts */
3428         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
3429         /* Restart the interface */
3430         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3431         return;
3432 }
3433 #endif
3434
3435 /**********************************************************************
3436  *
3437  *  Update the board statistics counters.
3438  *
3439  **********************************************************************/
3440 static void
3441 ixgbe_update_stats_counters(struct adapter *adapter)
3442 {
3443         struct ixgbe_hw *hw = &adapter->hw;
3444         u32 missed_rx = 0, bprc, lxon, lxoff, total;
3445         u64 total_missed_rx = 0;
3446
3447         adapter->stats.pf.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
3448         adapter->stats.pf.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
3449         adapter->stats.pf.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
3450         adapter->stats.pf.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
3451
3452         /*
3453         ** Note: these are for the 8 possible traffic classes,
3454         **       which in current implementation is unused,
3455         **       therefore only 0 should read real data.
3456         */
3457         for (int i = 0; i < 8; i++) {
3458                 u32 mp;
3459                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
3460                 /* missed_rx tallies misses for the gprc workaround */
3461                 missed_rx += mp;
3462                 /* global total per queue */
3463                 adapter->stats.pf.mpc[i] += mp;
3464                 /* total for stats display */
3465                 total_missed_rx += adapter->stats.pf.mpc[i];
3466                 if (hw->mac.type == ixgbe_mac_82598EB) {
3467                         adapter->stats.pf.rnbc[i] +=
3468                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
3469                         adapter->stats.pf.qbtc[i] +=
3470                             IXGBE_READ_REG(hw, IXGBE_QBTC(i));
3471                         adapter->stats.pf.qbrc[i] +=
3472                             IXGBE_READ_REG(hw, IXGBE_QBRC(i));
3473                         adapter->stats.pf.pxonrxc[i] +=
3474                             IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
3475                 } else
3476                         adapter->stats.pf.pxonrxc[i] +=
3477                             IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
3478                 adapter->stats.pf.pxontxc[i] +=
3479                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
3480                 adapter->stats.pf.pxofftxc[i] +=
3481                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
3482                 if (hw->mac.type != ixgbe_mac_X550EM_x)
3483                         adapter->stats.pf.pxoffrxc[i] +=
3484                             IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
3485                 adapter->stats.pf.pxon2offc[i] +=
3486                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
3487         }
3488         for (int i = 0; i < 16; i++) {
3489                 adapter->stats.pf.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
3490                 adapter->stats.pf.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
3491                 adapter->stats.pf.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
3492         }
3493         adapter->stats.pf.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
3494         adapter->stats.pf.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
3495         adapter->stats.pf.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
3496
3497         /* Hardware workaround, gprc counts missed packets */
3498         adapter->stats.pf.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
3499         adapter->stats.pf.gprc -= missed_rx;
3500
3501         if (hw->mac.type != ixgbe_mac_82598EB) {
3502                 adapter->stats.pf.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
3503                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
3504                 adapter->stats.pf.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
3505                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
3506                 adapter->stats.pf.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
3507                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
3508                 adapter->stats.pf.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
3509                 adapter->stats.pf.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
3510         } else {
3511                 adapter->stats.pf.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
3512                 adapter->stats.pf.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
3513                 /* 82598 only has a counter in the high register */
3514                 adapter->stats.pf.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
3515                 adapter->stats.pf.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
3516                 adapter->stats.pf.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
3517         }
3518
3519         /*
3520          * Workaround: mprc hardware is incorrectly counting
3521          * broadcasts, so for now we subtract those.
3522          */
3523         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
3524         adapter->stats.pf.bprc += bprc;
3525         adapter->stats.pf.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
3526         if (hw->mac.type == ixgbe_mac_82598EB)
3527                 adapter->stats.pf.mprc -= bprc;
3528
3529         adapter->stats.pf.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
3530         adapter->stats.pf.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
3531         adapter->stats.pf.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
3532         adapter->stats.pf.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
3533         adapter->stats.pf.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
3534         adapter->stats.pf.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
3535
3536         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
3537         adapter->stats.pf.lxontxc += lxon;
3538         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
3539         adapter->stats.pf.lxofftxc += lxoff;
3540         total = lxon + lxoff;
3541
3542         adapter->stats.pf.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
3543         adapter->stats.pf.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
3544         adapter->stats.pf.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
3545         adapter->stats.pf.gptc -= total;
3546         adapter->stats.pf.mptc -= total;
3547         adapter->stats.pf.ptc64 -= total;
3548         adapter->stats.pf.gotc -= total * ETHER_MIN_LEN;
3549
3550         adapter->stats.pf.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
3551         adapter->stats.pf.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
3552         adapter->stats.pf.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
3553         adapter->stats.pf.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
3554         adapter->stats.pf.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
3555         adapter->stats.pf.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
3556         adapter->stats.pf.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
3557         adapter->stats.pf.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
3558         adapter->stats.pf.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
3559         adapter->stats.pf.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
3560         adapter->stats.pf.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
3561         adapter->stats.pf.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
3562         adapter->stats.pf.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
3563         adapter->stats.pf.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
3564         adapter->stats.pf.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
3565         adapter->stats.pf.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
3566         adapter->stats.pf.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
3567         adapter->stats.pf.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
3568         /* Only read FCOE on 82599 */
3569         if (hw->mac.type != ixgbe_mac_82598EB) {
3570                 adapter->stats.pf.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
3571                 adapter->stats.pf.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
3572                 adapter->stats.pf.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
3573                 adapter->stats.pf.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
3574                 adapter->stats.pf.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
3575         }
3576
3577         /* Fill out the OS statistics structure */
3578         IXGBE_SET_IPACKETS(adapter, adapter->stats.pf.gprc);
3579         IXGBE_SET_OPACKETS(adapter, adapter->stats.pf.gptc);
3580         IXGBE_SET_IBYTES(adapter, adapter->stats.pf.gorc);
3581         IXGBE_SET_OBYTES(adapter, adapter->stats.pf.gotc);
3582         IXGBE_SET_IMCASTS(adapter, adapter->stats.pf.mprc);
3583         IXGBE_SET_OMCASTS(adapter, adapter->stats.pf.mptc);
3584         IXGBE_SET_COLLISIONS(adapter, 0);
3585         IXGBE_SET_IQDROPS(adapter, total_missed_rx);
3586         IXGBE_SET_IERRORS(adapter, adapter->stats.pf.crcerrs
3587             + adapter->stats.pf.rlec);
3588 }
3589
3590 #if __FreeBSD_version >= 1100036
3591 static uint64_t
3592 ixgbe_get_counter(struct ifnet *ifp, ift_counter cnt)
3593 {
3594         struct adapter *adapter;
3595
3596         adapter = if_getsoftc(ifp);
3597
3598         switch (cnt) {
3599         case IFCOUNTER_IPACKETS:
3600                 return (adapter->ipackets);
3601         case IFCOUNTER_OPACKETS:
3602                 return (adapter->opackets);
3603         case IFCOUNTER_IBYTES:
3604                 return (adapter->ibytes);
3605         case IFCOUNTER_OBYTES:
3606                 return (adapter->obytes);
3607         case IFCOUNTER_IMCASTS:
3608                 return (adapter->imcasts);
3609         case IFCOUNTER_OMCASTS:
3610                 return (adapter->omcasts);
3611         case IFCOUNTER_COLLISIONS:
3612                 return (0);
3613         case IFCOUNTER_IQDROPS:
3614                 return (adapter->iqdrops);
3615         case IFCOUNTER_IERRORS:
3616                 return (adapter->ierrors);
3617         default:
3618                 return (if_get_counter_default(ifp, cnt));
3619         }
3620 }
3621 #endif
3622
3623 /** ixgbe_sysctl_tdh_handler - Handler function
3624  *  Retrieves the TDH value from the hardware
3625  */
3626 static int
3627 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
3628 {
3629         int error;
3630
3631         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
3632         if (!txr) return 0;
3633
3634         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
3635         error = sysctl_handle_int(oidp, &val, 0, req);
3636         if (error || !req->newptr)
3637                 return error;
3638         return 0;
3639 }
3640
3641 /** ixgbe_sysctl_tdt_handler - Handler function
3642  *  Retrieves the TDT value from the hardware
3643  */
3644 static int
3645 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
3646 {
3647         int error;
3648
3649         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
3650         if (!txr) return 0;
3651
3652         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
3653         error = sysctl_handle_int(oidp, &val, 0, req);
3654         if (error || !req->newptr)
3655                 return error;
3656         return 0;
3657 }
3658
3659 /** ixgbe_sysctl_rdh_handler - Handler function
3660  *  Retrieves the RDH value from the hardware
3661  */
3662 static int
3663 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
3664 {
3665         int error;
3666
3667         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
3668         if (!rxr) return 0;
3669
3670         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
3671         error = sysctl_handle_int(oidp, &val, 0, req);
3672         if (error || !req->newptr)
3673                 return error;
3674         return 0;
3675 }
3676
3677 /** ixgbe_sysctl_rdt_handler - Handler function
3678  *  Retrieves the RDT value from the hardware
3679  */
3680 static int
3681 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
3682 {
3683         int error;
3684
3685         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
3686         if (!rxr) return 0;
3687
3688         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
3689         error = sysctl_handle_int(oidp, &val, 0, req);
3690         if (error || !req->newptr)
3691                 return error;
3692         return 0;
3693 }
3694
3695 static int
3696 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
3697 {
3698         int error;
3699         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
3700         unsigned int reg, usec, rate;
3701
3702         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
3703         usec = ((reg & 0x0FF8) >> 3);
3704         if (usec > 0)
3705                 rate = 500000 / usec;
3706         else
3707                 rate = 0;
3708         error = sysctl_handle_int(oidp, &rate, 0, req);
3709         if (error || !req->newptr)
3710                 return error;
3711         reg &= ~0xfff; /* default, no limitation */
3712         ixgbe_max_interrupt_rate = 0;
3713         if (rate > 0 && rate < 500000) {
3714                 if (rate < 1000)
3715                         rate = 1000;
3716                 ixgbe_max_interrupt_rate = rate;
3717                 reg |= ((4000000/rate) & 0xff8 );
3718         }
3719         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
3720         return 0;
3721 }
3722
3723 /*
3724  * Add sysctl variables, one per statistic, to the system.
3725  */
3726 static void
3727 ixgbe_add_hw_stats(struct adapter *adapter)
3728 {
3729         device_t dev = adapter->dev;
3730
3731         struct tx_ring *txr = adapter->tx_rings;
3732         struct rx_ring *rxr = adapter->rx_rings;
3733
3734         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3735         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3736         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3737         struct ixgbe_hw_stats *stats = &adapter->stats.pf;
3738
3739         struct sysctl_oid *stat_node, *queue_node;
3740         struct sysctl_oid_list *stat_list, *queue_list;
3741
3742 #define QUEUE_NAME_LEN 32
3743         char namebuf[QUEUE_NAME_LEN];
3744
3745         /* Driver Statistics */
3746         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
3747                         CTLFLAG_RD, &adapter->dropped_pkts,
3748                         "Driver dropped packets");
3749         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
3750                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
3751                         "m_defrag() failed");
3752         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3753                         CTLFLAG_RD, &adapter->watchdog_events,
3754                         "Watchdog timeouts");
3755         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
3756                         CTLFLAG_RD, &adapter->vector_irq,
3757                         "Link MSIX IRQ Handled");
3758
3759         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3760                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
3761                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
3762                                             CTLFLAG_RD, NULL, "Queue Name");
3763                 queue_list = SYSCTL_CHILDREN(queue_node);
3764
3765                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
3766                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
3767                                 sizeof(&adapter->queues[i]),
3768                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
3769                                 "Interrupt Rate");
3770                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3771                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
3772                                 "irqs on this queue");
3773                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
3774                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
3775                                 ixgbe_sysctl_tdh_handler, "IU",
3776                                 "Transmit Descriptor Head");
3777                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
3778                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
3779                                 ixgbe_sysctl_tdt_handler, "IU",
3780                                 "Transmit Descriptor Tail");
3781                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
3782                                 CTLFLAG_RD, &txr->tso_tx,
3783                                 "TSO");
3784                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
3785                                 CTLFLAG_RD, &txr->no_tx_dma_setup,
3786                                 "Driver tx dma failure in xmit");
3787                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3788                                 CTLFLAG_RD, &txr->no_desc_avail,
3789                                 "Queue No Descriptor Available");
3790                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3791                                 CTLFLAG_RD, &txr->total_packets,
3792                                 "Queue Packets Transmitted");
3793         }
3794
3795         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3796                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
3797                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
3798                                             CTLFLAG_RD, NULL, "Queue Name");
3799                 queue_list = SYSCTL_CHILDREN(queue_node);
3800
3801                 struct lro_ctrl *lro = &rxr->lro;
3802
3803                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
3804                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
3805                                             CTLFLAG_RD, NULL, "Queue Name");
3806                 queue_list = SYSCTL_CHILDREN(queue_node);
3807
3808                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
3809                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
3810                                 ixgbe_sysctl_rdh_handler, "IU",
3811                                 "Receive Descriptor Head");
3812                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
3813                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
3814                                 ixgbe_sysctl_rdt_handler, "IU",
3815                                 "Receive Descriptor Tail");
3816                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3817                                 CTLFLAG_RD, &rxr->rx_packets,
3818                                 "Queue Packets Received");
3819                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3820                                 CTLFLAG_RD, &rxr->rx_bytes,
3821                                 "Queue Bytes Received");
3822                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
3823                                 CTLFLAG_RD, &rxr->rx_copies,
3824                                 "Copied RX Frames");
3825                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
3826                                 CTLFLAG_RD, &lro->lro_queued, 0,
3827                                 "LRO Queued");
3828                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
3829                                 CTLFLAG_RD, &lro->lro_flushed, 0,
3830                                 "LRO Flushed");
3831         }
3832
3833         /* MAC stats get the own sub node */
3834
3835         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
3836                                     CTLFLAG_RD, NULL, "MAC Statistics");
3837         stat_list = SYSCTL_CHILDREN(stat_node);
3838
3839         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
3840                         CTLFLAG_RD, &stats->crcerrs,
3841                         "CRC Errors");
3842         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
3843                         CTLFLAG_RD, &stats->illerrc,
3844                         "Illegal Byte Errors");
3845         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
3846                         CTLFLAG_RD, &stats->errbc,
3847                         "Byte Errors");
3848         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
3849                         CTLFLAG_RD, &stats->mspdc,
3850                         "MAC Short Packets Discarded");
3851         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
3852                         CTLFLAG_RD, &stats->mlfc,
3853                         "MAC Local Faults");
3854         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
3855                         CTLFLAG_RD, &stats->mrfc,
3856                         "MAC Remote Faults");
3857         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
3858                         CTLFLAG_RD, &stats->rlec,
3859                         "Receive Length Errors");
3860
3861         /* Flow Control stats */
3862         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
3863                         CTLFLAG_RD, &stats->lxontxc,
3864                         "Link XON Transmitted");
3865         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
3866                         CTLFLAG_RD, &stats->lxonrxc,
3867                         "Link XON Received");
3868         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
3869                         CTLFLAG_RD, &stats->lxofftxc,
3870                         "Link XOFF Transmitted");
3871         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
3872                         CTLFLAG_RD, &stats->lxoffrxc,
3873                         "Link XOFF Received");
3874
3875         /* Packet Reception Stats */
3876         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
3877                         CTLFLAG_RD, &stats->tor,
3878                         "Total Octets Received");
3879         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
3880                         CTLFLAG_RD, &stats->gorc,
3881                         "Good Octets Received");
3882         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
3883                         CTLFLAG_RD, &stats->tpr,
3884                         "Total Packets Received");
3885         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
3886                         CTLFLAG_RD, &stats->gprc,
3887                         "Good Packets Received");
3888         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
3889                         CTLFLAG_RD, &stats->mprc,
3890                         "Multicast Packets Received");
3891         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
3892                         CTLFLAG_RD, &stats->bprc,
3893                         "Broadcast Packets Received");
3894         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
3895                         CTLFLAG_RD, &stats->prc64,
3896                         "64 byte frames received ");
3897         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
3898                         CTLFLAG_RD, &stats->prc127,
3899                         "65-127 byte frames received");
3900         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
3901                         CTLFLAG_RD, &stats->prc255,
3902                         "128-255 byte frames received");
3903         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
3904                         CTLFLAG_RD, &stats->prc511,
3905                         "256-511 byte frames received");
3906         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
3907                         CTLFLAG_RD, &stats->prc1023,
3908                         "512-1023 byte frames received");
3909         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
3910                         CTLFLAG_RD, &stats->prc1522,
3911                         "1023-1522 byte frames received");
3912         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
3913                         CTLFLAG_RD, &stats->ruc,
3914                         "Receive Undersized");
3915         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
3916                         CTLFLAG_RD, &stats->rfc,
3917                         "Fragmented Packets Received ");
3918         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
3919                         CTLFLAG_RD, &stats->roc,
3920                         "Oversized Packets Received");
3921         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
3922                         CTLFLAG_RD, &stats->rjc,
3923                         "Received Jabber");
3924         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
3925                         CTLFLAG_RD, &stats->mngprc,
3926                         "Management Packets Received");
3927         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
3928                         CTLFLAG_RD, &stats->mngptc,
3929                         "Management Packets Dropped");
3930         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
3931                         CTLFLAG_RD, &stats->xec,
3932                         "Checksum Errors");
3933
3934         /* Packet Transmission Stats */
3935         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
3936                         CTLFLAG_RD, &stats->gotc,
3937                         "Good Octets Transmitted");
3938         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
3939                         CTLFLAG_RD, &stats->tpt,
3940                         "Total Packets Transmitted");
3941         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
3942                         CTLFLAG_RD, &stats->gptc,
3943                         "Good Packets Transmitted");
3944         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
3945                         CTLFLAG_RD, &stats->bptc,
3946                         "Broadcast Packets Transmitted");
3947         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
3948                         CTLFLAG_RD, &stats->mptc,
3949                         "Multicast Packets Transmitted");
3950         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
3951                         CTLFLAG_RD, &stats->mngptc,
3952                         "Management Packets Transmitted");
3953         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
3954                         CTLFLAG_RD, &stats->ptc64,
3955                         "64 byte frames transmitted ");
3956         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
3957                         CTLFLAG_RD, &stats->ptc127,
3958                         "65-127 byte frames transmitted");
3959         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
3960                         CTLFLAG_RD, &stats->ptc255,
3961                         "128-255 byte frames transmitted");
3962         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
3963                         CTLFLAG_RD, &stats->ptc511,
3964                         "256-511 byte frames transmitted");
3965         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
3966                         CTLFLAG_RD, &stats->ptc1023,
3967                         "512-1023 byte frames transmitted");
3968         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
3969                         CTLFLAG_RD, &stats->ptc1522,
3970                         "1024-1522 byte frames transmitted");
3971 }
3972
3973 /*
3974 ** Set flow control using sysctl:
3975 ** Flow control values:
3976 **      0 - off
3977 **      1 - rx pause
3978 **      2 - tx pause
3979 **      3 - full
3980 */
3981 static int
3982 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
3983 {
3984         int error, last;
3985         struct adapter *adapter = (struct adapter *) arg1;
3986
3987         last = adapter->fc;
3988         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
3989         if ((error) || (req->newptr == NULL))
3990                 return (error);
3991
3992         /* Don't bother if it's not changed */
3993         if (adapter->fc == last)
3994                 return (0);
3995
3996         switch (adapter->fc) {
3997                 case ixgbe_fc_rx_pause:
3998                 case ixgbe_fc_tx_pause:
3999                 case ixgbe_fc_full:
4000                         adapter->hw.fc.requested_mode = adapter->fc;
4001                         if (adapter->num_queues > 1)
4002                                 ixgbe_disable_rx_drop(adapter);
4003                         break;
4004                 case ixgbe_fc_none:
4005                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
4006                         if (adapter->num_queues > 1)
4007                                 ixgbe_enable_rx_drop(adapter);
4008                         break;
4009                 default:
4010                         adapter->fc = last;
4011                         return (EINVAL);
4012         }
4013         /* Don't autoneg if forcing a value */
4014         adapter->hw.fc.disable_fc_autoneg = TRUE;
4015         ixgbe_fc_enable(&adapter->hw);
4016         return error;
4017 }
4018
4019 /*
4020 ** Control advertised link speed:
4021 **      Flags:
4022 **      0x1 - advertise 100 Mb
4023 **      0x2 - advertise 1G
4024 **      0x4 - advertise 10G
4025 */
4026 static int
4027 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
4028 {
4029         int                     error = 0, requested;
4030         struct adapter          *adapter;
4031         device_t                dev;
4032         struct ixgbe_hw         *hw;
4033         ixgbe_link_speed        speed = 0;
4034
4035         adapter = (struct adapter *) arg1;
4036         dev = adapter->dev;
4037         hw = &adapter->hw;
4038
4039         requested = adapter->advertise;
4040         error = sysctl_handle_int(oidp, &requested, 0, req);
4041         if ((error) || (req->newptr == NULL))
4042                 return (error);
4043
4044         /* Checks to validate new value */
4045         if (adapter->advertise == requested) /* no change */
4046                 return (0);
4047
4048         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
4049             (hw->phy.multispeed_fiber))) {
4050                 device_printf(dev,
4051                     "Advertised speed can only be set on copper or "
4052                     "multispeed fiber media types.\n");
4053                 return (EINVAL);
4054         }
4055
4056         if (requested < 0x1 || requested > 0x7) {
4057                 device_printf(dev,
4058                     "Invalid advertised speed; valid modes are 0x1 through 0x7\n");
4059                 return (EINVAL);
4060         }
4061
4062         if ((requested & 0x1)
4063             && (hw->mac.type != ixgbe_mac_X540)
4064             && (hw->mac.type != ixgbe_mac_X550)) {
4065                 device_printf(dev, "Set Advertise: 100Mb on X540/X550 only\n");
4066                 return (EINVAL);
4067         }
4068
4069         /* Set new value and report new advertised mode */
4070         if (requested & 0x1)
4071                 speed |= IXGBE_LINK_SPEED_100_FULL;
4072         if (requested & 0x2)
4073                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
4074         if (requested & 0x4)
4075                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
4076
4077         hw->mac.autotry_restart = TRUE;
4078         hw->mac.ops.setup_link(hw, speed, TRUE);
4079         adapter->advertise = requested;
4080
4081         return (error);
4082 }
4083
4084 /*
4085 ** Thermal Shutdown Trigger
4086 **   - cause a Thermal Overtemp IRQ
4087 **   - this now requires firmware enabling
4088 */
4089 static int
4090 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
4091 {
4092         int             error, fire = 0;
4093         struct adapter  *adapter = (struct adapter *) arg1;
4094         struct ixgbe_hw *hw = &adapter->hw;
4095
4096
4097         if (hw->mac.type < ixgbe_mac_X540)
4098                 return (0);
4099
4100         error = sysctl_handle_int(oidp, &fire, 0, req);
4101         if ((error) || (req->newptr == NULL))
4102                 return (error);
4103
4104         if (fire) {
4105                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
4106                 reg |= IXGBE_EICR_TS;
4107                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
4108         }
4109
4110         return (0);
4111 }
4112
4113 /*
4114 ** Enable the hardware to drop packets when the buffer is
4115 ** full. This is useful when multiqueue,so that no single
4116 ** queue being full stalls the entire RX engine. We only
4117 ** enable this when Multiqueue AND when Flow Control is
4118 ** disabled.
4119 */
4120 static void
4121 ixgbe_enable_rx_drop(struct adapter *adapter)
4122 {
4123         struct ixgbe_hw *hw = &adapter->hw;
4124
4125         for (int i = 0; i < adapter->num_queues; i++) {
4126                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4127                 srrctl |= IXGBE_SRRCTL_DROP_EN;
4128                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4129         }
4130 }
4131
4132 static void
4133 ixgbe_disable_rx_drop(struct adapter *adapter)
4134 {
4135         struct ixgbe_hw *hw = &adapter->hw;
4136
4137         for (int i = 0; i < adapter->num_queues; i++) {
4138                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4139                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
4140                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4141         }
4142 }
4143
4144 static void
4145 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
4146 {
4147         u32 mask;
4148
4149         switch (adapter->hw.mac.type) {
4150         case ixgbe_mac_82598EB:
4151                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
4152                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
4153                 break;
4154         case ixgbe_mac_82599EB:
4155         case ixgbe_mac_X540:
4156         case ixgbe_mac_X550:
4157                 mask = (queues & 0xFFFFFFFF);
4158                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
4159                 mask = (queues >> 32);
4160                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
4161                 break;
4162         default:
4163                 break;
4164         }
4165 }
4166
4167