sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2013, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #endif
  39
  40 #include "ixgbe.h"
  41
  42 /*********************************************************************
  43  *  Set this to one to display debug statistics
  44  *********************************************************************/
  45 int             ixgbe_display_debug_stats = 0;
  46
  47 /*********************************************************************
  48  *  Driver version
  49  *********************************************************************/
  50 char ixgbe_driver_version[] = "2.5.0 - 8";
  51
  52 /*********************************************************************
  53  *  PCI Device ID Table
  54  *
  55  *  Used by probe to select devices to load on
  56  *  Last field stores an index into ixgbe_strings
  57  *  Last entry must be all 0s
  58  *
  59  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  60  *********************************************************************/
  61
  62 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  63 {
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  86         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  87         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  88         /* required last entry */
  89         {0, 0, 0, 0, 0}
  90 };
  91
  92 /*********************************************************************
  93  *  Table of branding strings
  94  *********************************************************************/
  95
  96 static char    *ixgbe_strings[] = {
  97         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  98 };
  99
 100 /*********************************************************************
 101  *  Function prototypes
 102  *********************************************************************/
 103 static int      ixgbe_probe(device_t);
 104 static int      ixgbe_attach(device_t);
 105 static int      ixgbe_detach(device_t);
 106 static int      ixgbe_shutdown(device_t);
 107 #ifdef IXGBE_LEGACY_TX
 108 static void     ixgbe_start(struct ifnet *);
 109 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 110 #else /* ! IXGBE_LEGACY_TX */
 111 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 112 static int      ixgbe_mq_start_locked(struct ifnet *,
 113                     struct tx_ring *, struct mbuf *);
 114 static void     ixgbe_qflush(struct ifnet *);
 115 static void     ixgbe_deferred_mq_start(void *, int);
 116 #endif /* IXGBE_LEGACY_TX */
 117 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 118 static void     ixgbe_init(void *);
 119 static void     ixgbe_init_locked(struct adapter *);
 120 static void     ixgbe_stop(void *);
 121 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 122 static int      ixgbe_media_change(struct ifnet *);
 123 static void     ixgbe_identify_hardware(struct adapter *);
 124 static int      ixgbe_allocate_pci_resources(struct adapter *);
 125 static int      ixgbe_allocate_msix(struct adapter *);
 126 static int      ixgbe_allocate_legacy(struct adapter *);
 127 static int      ixgbe_allocate_queues(struct adapter *);
 128 static int      ixgbe_setup_msix(struct adapter *);
 129 static void     ixgbe_free_pci_resources(struct adapter *);
 130 static void     ixgbe_local_timer(void *);
 131 static int      ixgbe_setup_interface(device_t, struct adapter *);
 132 static void     ixgbe_config_link(struct adapter *);
 133
 134 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 135 static int      ixgbe_setup_transmit_structures(struct adapter *);
 136 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 137 static void     ixgbe_initialize_transmit_units(struct adapter *);
 138 static void     ixgbe_free_transmit_structures(struct adapter *);
 139 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 140
 141 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 142 static int      ixgbe_setup_receive_structures(struct adapter *);
 143 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 144 static void     ixgbe_initialize_receive_units(struct adapter *);
 145 static void     ixgbe_free_receive_structures(struct adapter *);
 146 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 147 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 148
 149 static void     ixgbe_enable_intr(struct adapter *);
 150 static void     ixgbe_disable_intr(struct adapter *);
 151 static void     ixgbe_update_stats_counters(struct adapter *);
 152 static bool     ixgbe_txeof(struct tx_ring *);
 153 static bool     ixgbe_rxeof(struct ix_queue *);
 154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 155 static void     ixgbe_set_promisc(struct adapter *);
 156 static void     ixgbe_set_multi(struct adapter *);
 157 static void     ixgbe_update_link_status(struct adapter *);
 158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 164                     struct ixgbe_dma_alloc *, int);
 165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 166 static int      ixgbe_tx_ctx_setup(struct tx_ring *,
 167                     struct mbuf *, u32 *, u32 *);
 168 static int      ixgbe_tso_setup(struct tx_ring *,
 169                     struct mbuf *, u32 *, u32 *);
 170 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 171 static void     ixgbe_configure_ivars(struct adapter *);
 172 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 173
 174 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 175 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 176 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 177
 178 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 179
 180 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 181 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 182                     struct mbuf *, u32);
 183
 184 static void     ixgbe_enable_rx_drop(struct adapter *);
 185 static void     ixgbe_disable_rx_drop(struct adapter *);
 186
 187 /* Support for pluggable optic modules */
 188 static bool     ixgbe_sfp_probe(struct adapter *);
 189 static void     ixgbe_setup_optics(struct adapter *);
 190
 191 /* Legacy (single vector interrupt handler */
 192 static void     ixgbe_legacy_irq(void *);
 193
 194 /* The MSI/X Interrupt handlers */
 195 static void     ixgbe_msix_que(void *);
 196 static void     ixgbe_msix_link(void *);
 197
 198 /* Deferred interrupt tasklets */
 199 static void     ixgbe_handle_que(void *, int);
 200 static void     ixgbe_handle_link(void *, int);
 201 static void     ixgbe_handle_msf(void *, int);
 202 static void     ixgbe_handle_mod(void *, int);
 203
 204 #ifdef IXGBE_FDIR
 205 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 206 static void     ixgbe_reinit_fdir(void *, int);
 207 #endif
 208
 209 /*********************************************************************
 210  *  FreeBSD Device Interface Entry Points
 211  *********************************************************************/
 212
 213 static device_method_t ixgbe_methods[] = {
 214         /* Device interface */
 215         DEVMETHOD(device_probe, ixgbe_probe),
 216         DEVMETHOD(device_attach, ixgbe_attach),
 217         DEVMETHOD(device_detach, ixgbe_detach),
 218         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 219
 220         DEVMETHOD_END
 221 };
 222
 223 static driver_t ixgbe_driver = {
 224         "ix", ixgbe_methods, sizeof(struct adapter),
 225 };
 226
 227 devclass_t ixgbe_devclass;
 228 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 229
 230 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 231 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 232
 233 /*
 234 ** TUNEABLE PARAMETERS:
 235 */
 236
 237 /*
 238 ** AIM: Adaptive Interrupt Moderation
 239 ** which means that the interrupt rate
 240 ** is varied over time based on the
 241 ** traffic for that interrupt vector
 242 */
 243 static int ixgbe_enable_aim = TRUE;
 244 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 245
 246 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 247 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 248
 249 /* How many packets rxeof tries to clean at a time */
 250 static int ixgbe_rx_process_limit = 256;
 251 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 252
 253 /* How many packets txeof tries to clean at a time */
 254 static int ixgbe_tx_process_limit = 256;
 255 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
 256
 257 /*
 258 ** Smart speed setting, default to on
 259 ** this only works as a compile option
 260 ** right now as its during attach, set
 261 ** this to 'ixgbe_smart_speed_off' to
 262 ** disable.
 263 */
 264 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 265
 266 /*
 267  * MSIX should be the default for best performance,
 268  * but this allows it to be forced off for testing.
 269  */
 270 static int ixgbe_enable_msix = 1;
 271 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 272
 273 /*
 274  * Number of Queues, can be set to 0,
 275  * it then autoconfigures based on the
 276  * number of cpus with a max of 8. This
 277  * can be overriden manually here.
 278  */
 279 static int ixgbe_num_queues = 0;
 280 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 281
 282 /*
 283 ** Number of TX descriptors per ring,
 284 ** setting higher than RX as this seems
 285 ** the better performing choice.
 286 */
 287 static int ixgbe_txd = PERFORM_TXD;
 288 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 289
 290 /* Number of RX descriptors per ring */
 291 static int ixgbe_rxd = PERFORM_RXD;
 292 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 293
 294 /*
 295 ** HW RSC control:
 296 **  this feature only works with
 297 **  IPv4, and only on 82599 and later.
 298 **  Also this will cause IP forwarding to
 299 **  fail and that can't be controlled by
 300 **  the stack as LRO can. For all these
 301 **  reasons I've deemed it best to leave
 302 **  this off and not bother with a tuneable
 303 **  interface, this would need to be compiled
 304 **  to enable.
 305 */
 306 static bool ixgbe_rsc_enable = FALSE;
 307
 308 /* Keep running tab on them for sanity check */
 309 static int ixgbe_total_ports;
 310
 311 #ifdef IXGBE_FDIR
 312 /*
 313 ** For Flow Director: this is the
 314 ** number of TX packets we sample
 315 ** for the filter pool, this means
 316 ** every 20th packet will be probed.
 317 **
 318 ** This feature can be disabled by
 319 ** setting this to 0.
 320 */
 321 static int atr_sample_rate = 20;
 322 /*
 323 ** Flow Director actually 'steals'
 324 ** part of the packet buffer as its
 325 ** filter pool, this variable controls
 326 ** how much it uses:
 327 **  0 = 64K, 1 = 128K, 2 = 256K
 328 */
 329 static int fdir_pballoc = 1;
 330 #endif
 331
 332 #ifdef DEV_NETMAP
 333 /*
 334  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 335  * be a reference on how to implement netmap support in a driver.
 336  * Additional comments are in ixgbe_netmap.h .
 337  *
 338  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 339  * that extend the standard driver.
 340  */
 341 #include <dev/netmap/ixgbe_netmap.h>
 342 #endif /* DEV_NETMAP */
 343
 344 /*********************************************************************
 345  *  Device identification routine
 346  *
 347  *  ixgbe_probe determines if the driver should be loaded on
 348  *  adapter based on PCI vendor/device id of the adapter.
 349  *
 350  *  return BUS_PROBE_DEFAULT on success, positive on failure
 351  *********************************************************************/
 352
 353 static int
 354 ixgbe_probe(device_t dev)
 355 {
 356         ixgbe_vendor_info_t *ent;
 357
 358         u16     pci_vendor_id = 0;
 359         u16     pci_device_id = 0;
 360         u16     pci_subvendor_id = 0;
 361         u16     pci_subdevice_id = 0;
 362         char    adapter_name[256];
 363
 364         INIT_DEBUGOUT("ixgbe_probe: begin");
 365
 366         pci_vendor_id = pci_get_vendor(dev);
 367         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 368                 return (ENXIO);
 369
 370         pci_device_id = pci_get_device(dev);
 371         pci_subvendor_id = pci_get_subvendor(dev);
 372         pci_subdevice_id = pci_get_subdevice(dev);
 373
 374         ent = ixgbe_vendor_info_array;
 375         while (ent->vendor_id != 0) {
 376                 if ((pci_vendor_id == ent->vendor_id) &&
 377                     (pci_device_id == ent->device_id) &&
 378
 379                     ((pci_subvendor_id == ent->subvendor_id) ||
 380                      (ent->subvendor_id == 0)) &&
 381
 382                     ((pci_subdevice_id == ent->subdevice_id) ||
 383                      (ent->subdevice_id == 0))) {
 384                         sprintf(adapter_name, "%s, Version - %s",
 385                                 ixgbe_strings[ent->index],
 386                                 ixgbe_driver_version);
 387                         device_set_desc_copy(dev, adapter_name);
 388                         ++ixgbe_total_ports;
 389                         return (BUS_PROBE_DEFAULT);
 390                 }
 391                 ent++;
 392         }
 393         return (ENXIO);
 394 }
 395
 396 /*********************************************************************
 397  *  Device initialization routine
 398  *
 399  *  The attach entry point is called when the driver is being loaded.
 400  *  This routine identifies the type of hardware, allocates all resources
 401  *  and initializes the hardware.
 402  *
 403  *  return 0 on success, positive on failure
 404  *********************************************************************/
 405
 406 static int
 407 ixgbe_attach(device_t dev)
 408 {
 409         struct adapter *adapter;
 410         struct ixgbe_hw *hw;
 411         int             error = 0;
 412         u16             csum;
 413         u32             ctrl_ext;
 414
 415         INIT_DEBUGOUT("ixgbe_attach: begin");
 416
 417         /* Allocate, clear, and link in our adapter structure */
 418         adapter = device_get_softc(dev);
 419         adapter->dev = adapter->osdep.dev = dev;
 420         hw = &adapter->hw;
 421
 422         /* Core Lock Init*/
 423         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 424
 425         /* SYSCTL APIs */
 426
 427         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 428                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 429                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 430                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 431
 432         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 433                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 434                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 435                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 436
 437         /*
 438         ** Allow a kind of speed control by forcing the autoneg
 439         ** advertised speed list to only a certain value, this
 440         ** supports 1G on 82599 devices, and 100Mb on x540.
 441         */
 442         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 443                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 444                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 445                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 446
 447         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 448                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 449                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 450                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 451
 452         /* Set up the timer callout */
 453         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 454
 455         /* Determine hardware revision */
 456         ixgbe_identify_hardware(adapter);
 457
 458         /* Do base PCI setup - map BAR0 */
 459         if (ixgbe_allocate_pci_resources(adapter)) {
 460                 device_printf(dev, "Allocation of PCI resources failed\n");
 461                 error = ENXIO;
 462                 goto err_out;
 463         }
 464
 465         /* Do descriptor calc and sanity checks */
 466         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 467             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 468                 device_printf(dev, "TXD config issue, using default!\n");
 469                 adapter->num_tx_desc = DEFAULT_TXD;
 470         } else
 471                 adapter->num_tx_desc = ixgbe_txd;
 472
 473         /*
 474         ** With many RX rings it is easy to exceed the
 475         ** system mbuf allocation. Tuning nmbclusters
 476         ** can alleviate this.
 477         */
 478         if (nmbclusters > 0 ) {
 479                 int s;
 480                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 481                 if (s > nmbclusters) {
 482                         device_printf(dev, "RX Descriptors exceed "
 483                             "system mbuf max, using default instead!\n");
 484                         ixgbe_rxd = DEFAULT_RXD;
 485                 }
 486         }
 487
 488         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 489             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 490                 device_printf(dev, "RXD config issue, using default!\n");
 491                 adapter->num_rx_desc = DEFAULT_RXD;
 492         } else
 493                 adapter->num_rx_desc = ixgbe_rxd;
 494
 495         /* Allocate our TX/RX Queues */
 496         if (ixgbe_allocate_queues(adapter)) {
 497                 error = ENOMEM;
 498                 goto err_out;
 499         }
 500
 501         /* Allocate multicast array memory. */
 502         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 503             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 504         if (adapter->mta == NULL) {
 505                 device_printf(dev, "Can not allocate multicast setup array\n");
 506                 error = ENOMEM;
 507                 goto err_late;
 508         }
 509
 510         /* Initialize the shared code */
 511         error = ixgbe_init_shared_code(hw);
 512         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 513                 /*
 514                 ** No optics in this port, set up
 515                 ** so the timer routine will probe
 516                 ** for later insertion.
 517                 */
 518                 adapter->sfp_probe = TRUE;
 519                 error = 0;
 520         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 521                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 522                 error = EIO;
 523                 goto err_late;
 524         } else if (error) {
 525                 device_printf(dev,"Unable to initialize the shared code\n");
 526                 error = EIO;
 527                 goto err_late;
 528         }
 529
 530         /* Make sure we have a good EEPROM before we read from it */
 531         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 532                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 533                 error = EIO;
 534                 goto err_late;
 535         }
 536
 537         error = ixgbe_init_hw(hw);
 538         switch (error) {
 539         case IXGBE_ERR_EEPROM_VERSION:
 540                 device_printf(dev, "This device is a pre-production adapter/"
 541                     "LOM.  Please be aware there may be issues associated "
 542                     "with your hardware.\n If you are experiencing problems "
 543                     "please contact your Intel or hardware representative "
 544                     "who provided you with this hardware.\n");
 545                 break;
 546         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 547                 device_printf(dev,"Unsupported SFP+ Module\n");
 548                 error = EIO;
 549                 goto err_late;
 550         case IXGBE_ERR_SFP_NOT_PRESENT:
 551                 device_printf(dev,"No SFP+ Module found\n");
 552                 /* falls thru */
 553         default:
 554                 break;
 555         }
 556
 557         /* Detect and set physical type */
 558         ixgbe_setup_optics(adapter);
 559
 560         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 561                 error = ixgbe_allocate_msix(adapter);
 562         else
 563                 error = ixgbe_allocate_legacy(adapter);
 564         if (error)
 565                 goto err_late;
 566
 567         /* Setup OS specific network interface */
 568         if (ixgbe_setup_interface(dev, adapter) != 0)
 569                 goto err_late;
 570
 571         /* Initialize statistics */
 572         ixgbe_update_stats_counters(adapter);
 573
 574         /* Register for VLAN events */
 575         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 576             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 577         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 578             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 579
 580         /* Print PCIE bus type/speed/width info */
 581         ixgbe_get_bus_info(hw);
 582         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 583             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 584             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 585             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 586             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 587             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 588             ("Unknown"));
 589
 590         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 591             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 592                 device_printf(dev, "PCI-Express bandwidth available"
 593                     " for this card\n     is not sufficient for"
 594                     " optimal performance.\n");
 595                 device_printf(dev, "For optimal performance a x8 "
 596                     "PCIE, or x4 PCIE 2 slot is required.\n");
 597         }
 598
 599         /* let hardware know driver is loaded */
 600         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 601         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 602         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 603
 604         ixgbe_add_hw_stats(adapter);
 605
 606 #ifdef DEV_NETMAP
 607         ixgbe_netmap_attach(adapter);
 608 #endif /* DEV_NETMAP */
 609         INIT_DEBUGOUT("ixgbe_attach: end");
 610         return (0);
 611 err_late:
 612         ixgbe_free_transmit_structures(adapter);
 613         ixgbe_free_receive_structures(adapter);
 614 err_out:
 615         if (adapter->ifp != NULL)
 616                 if_free(adapter->ifp);
 617         ixgbe_free_pci_resources(adapter);
 618         free(adapter->mta, M_DEVBUF);
 619         return (error);
 620
 621 }
 622
 623 /*********************************************************************
 624  *  Device removal routine
 625  *
 626  *  The detach entry point is called when the driver is being removed.
 627  *  This routine stops the adapter and deallocates all the resources
 628  *  that were allocated for driver operation.
 629  *
 630  *  return 0 on success, positive on failure
 631  *********************************************************************/
 632
 633 static int
 634 ixgbe_detach(device_t dev)
 635 {
 636         struct adapter *adapter = device_get_softc(dev);
 637         struct ix_queue *que = adapter->queues;
 638         struct tx_ring *txr = adapter->tx_rings;
 639         u32     ctrl_ext;
 640
 641         INIT_DEBUGOUT("ixgbe_detach: begin");
 642
 643         /* Make sure VLANS are not using driver */
 644         if (adapter->ifp->if_vlantrunk != NULL) {
 645                 device_printf(dev,"Vlan in use, detach first\n");
 646                 return (EBUSY);
 647         }
 648
 649         IXGBE_CORE_LOCK(adapter);
 650         ixgbe_stop(adapter);
 651         IXGBE_CORE_UNLOCK(adapter);
 652
 653         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 654                 if (que->tq) {
 655 #ifdef IXGBE_LEGACY_TX
 656                         taskqueue_drain(que->tq, &txr->txq_task);
 657 #endif
 658                         taskqueue_drain(que->tq, &que->que_task);
 659                         taskqueue_free(que->tq);
 660                 }
 661         }
 662
 663         /* Drain the Link queue */
 664         if (adapter->tq) {
 665                 taskqueue_drain(adapter->tq, &adapter->link_task);
 666                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 667                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 668 #ifdef IXGBE_FDIR
 669                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 670 #endif
 671                 taskqueue_free(adapter->tq);
 672         }
 673
 674         /* let hardware know driver is unloading */
 675         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 676         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 677         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 678
 679         /* Unregister VLAN events */
 680         if (adapter->vlan_attach != NULL)
 681                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 682         if (adapter->vlan_detach != NULL)
 683                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 684
 685         ether_ifdetach(adapter->ifp);
 686         callout_drain(&adapter->timer);
 687 #ifdef DEV_NETMAP
 688         netmap_detach(adapter->ifp);
 689 #endif /* DEV_NETMAP */
 690         ixgbe_free_pci_resources(adapter);
 691         bus_generic_detach(dev);
 692         if_free(adapter->ifp);
 693
 694         ixgbe_free_transmit_structures(adapter);
 695         ixgbe_free_receive_structures(adapter);
 696         free(adapter->mta, M_DEVBUF);
 697
 698         IXGBE_CORE_LOCK_DESTROY(adapter);
 699         return (0);
 700 }
 701
 702 /*********************************************************************
 703  *
 704  *  Shutdown entry point
 705  *
 706  **********************************************************************/
 707
 708 static int
 709 ixgbe_shutdown(device_t dev)
 710 {
 711         struct adapter *adapter = device_get_softc(dev);
 712         IXGBE_CORE_LOCK(adapter);
 713         ixgbe_stop(adapter);
 714         IXGBE_CORE_UNLOCK(adapter);
 715         return (0);
 716 }
 717
 718
 719 #ifdef IXGBE_LEGACY_TX
 720 /*********************************************************************
 721  *  Transmit entry point
 722  *
 723  *  ixgbe_start is called by the stack to initiate a transmit.
 724  *  The driver will remain in this routine as long as there are
 725  *  packets to transmit and transmit resources are available.
 726  *  In case resources are not available stack is notified and
 727  *  the packet is requeued.
 728  **********************************************************************/
 729
 730 static void
 731 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 732 {
 733         struct mbuf    *m_head;
 734         struct adapter *adapter = txr->adapter;
 735
 736         IXGBE_TX_LOCK_ASSERT(txr);
 737
 738         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 739                 return;
 740         if (!adapter->link_active)
 741                 return;
 742
 743         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 744                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 745                         break;
 746
 747                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 748                 if (m_head == NULL)
 749                         break;
 750
 751                 if (ixgbe_xmit(txr, &m_head)) {
 752                         if (m_head != NULL)
 753                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 754                         break;
 755                 }
 756                 /* Send a copy of the frame to the BPF listener */
 757                 ETHER_BPF_MTAP(ifp, m_head);
 758
 759                 /* Set watchdog on */
 760                 txr->watchdog_time = ticks;
 761                 txr->queue_status = IXGBE_QUEUE_WORKING;
 762
 763         }
 764         return;
 765 }
 766
 767 /*
 768  * Legacy TX start - called by the stack, this
 769  * always uses the first tx ring, and should
 770  * not be used with multiqueue tx enabled.
 771  */
 772 static void
 773 ixgbe_start(struct ifnet *ifp)
 774 {
 775         struct adapter *adapter = ifp->if_softc;
 776         struct tx_ring  *txr = adapter->tx_rings;
 777
 778         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 779                 IXGBE_TX_LOCK(txr);
 780                 ixgbe_start_locked(txr, ifp);
 781                 IXGBE_TX_UNLOCK(txr);
 782         }
 783         return;
 784 }
 785
 786 #else /* ! IXGBE_LEGACY_TX */
 787
 788 /*
 789 ** Multiqueue Transmit driver
 790 **
 791 */
 792 static int
 793 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 794 {
 795         struct adapter  *adapter = ifp->if_softc;
 796         struct ix_queue *que;
 797         struct tx_ring  *txr;
 798         int             i = 0, err = 0;
 799
 800         /* Which queue to use */
 801         if ((m->m_flags & M_FLOWID) != 0)
 802                 i = m->m_pkthdr.flowid % adapter->num_queues;
 803         else
 804                 i = curcpu % adapter->num_queues;
 805
 806         txr = &adapter->tx_rings[i];
 807         que = &adapter->queues[i];
 808
 809         if (IXGBE_TX_TRYLOCK(txr)) {
 810                 err = ixgbe_mq_start_locked(ifp, txr, m);
 811                 IXGBE_TX_UNLOCK(txr);
 812         } else {
 813                 err = drbr_enqueue(ifp, txr->br, m);
 814                 taskqueue_enqueue(que->tq, &txr->txq_task);
 815         }
 816
 817         return (err);
 818 }
 819
 820 static int
 821 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 822 {
 823         struct adapter  *adapter = txr->adapter;
 824         struct mbuf     *next;
 825         int             enqueued, err = 0;
 826
 827         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 828             adapter->link_active == 0) {
 829                 if (m != NULL)
 830                         err = drbr_enqueue(ifp, txr->br, m);
 831                 return (err);
 832         }
 833
 834         enqueued = 0;
 835         if (m == NULL) {
 836                 next = drbr_dequeue(ifp, txr->br);
 837         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 838                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 839                         return (err);
 840                 next = drbr_dequeue(ifp, txr->br);
 841         } else
 842                 next = m;
 843
 844         /* Process the queue */
 845         while (next != NULL) {
 846                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 847                         if (next != NULL)
 848                                 err = drbr_enqueue(ifp, txr->br, next);
 849                         break;
 850                 }
 851                 enqueued++;
 852                 ifp->if_obytes += next->m_pkthdr.len;
 853                 if (next->m_flags & M_MCAST)
 854                         ifp->if_omcasts++;
 855                 /* Send a copy of the frame to the BPF listener */
 856                 ETHER_BPF_MTAP(ifp, next);
 857                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 858                         break;
 859                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 860                         ixgbe_txeof(txr);
 861                 next = drbr_dequeue(ifp, txr->br);
 862         }
 863
 864         if (enqueued > 0) {
 865                 /* Set watchdog on */
 866                 txr->queue_status = IXGBE_QUEUE_WORKING;
 867                 txr->watchdog_time = ticks;
 868         }
 869
 870         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 871                 ixgbe_txeof(txr);
 872
 873         return (err);
 874 }
 875
 876 /*
 877  * Called from a taskqueue to drain queued transmit packets.
 878  */
 879 static void
 880 ixgbe_deferred_mq_start(void *arg, int pending)
 881 {
 882         struct tx_ring *txr = arg;
 883         struct adapter *adapter = txr->adapter;
 884         struct ifnet *ifp = adapter->ifp;
 885
 886         IXGBE_TX_LOCK(txr);
 887         if (!drbr_empty(ifp, txr->br))
 888                 ixgbe_mq_start_locked(ifp, txr, NULL);
 889         IXGBE_TX_UNLOCK(txr);
 890 }
 891
 892 /*
 893 ** Flush all ring buffers
 894 */
 895 static void
 896 ixgbe_qflush(struct ifnet *ifp)
 897 {
 898         struct adapter  *adapter = ifp->if_softc;
 899         struct tx_ring  *txr = adapter->tx_rings;
 900         struct mbuf     *m;
 901
 902         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 903                 IXGBE_TX_LOCK(txr);
 904                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 905                         m_freem(m);
 906                 IXGBE_TX_UNLOCK(txr);
 907         }
 908         if_qflush(ifp);
 909 }
 910 #endif /* IXGBE_LEGACY_TX */
 911
 912 /*********************************************************************
 913  *  Ioctl entry point
 914  *
 915  *  ixgbe_ioctl is called when the user wants to configure the
 916  *  interface.
 917  *
 918  *  return 0 on success, positive on failure
 919  **********************************************************************/
 920
 921 static int
 922 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 923 {
 924         struct adapter  *adapter = ifp->if_softc;
 925         struct ixgbe_hw *hw = &adapter->hw;
 926         struct ifreq    *ifr = (struct ifreq *) data;
 927 #if defined(INET) || defined(INET6)
 928         struct ifaddr *ifa = (struct ifaddr *)data;
 929         bool            avoid_reset = FALSE;
 930 #endif
 931         int             error = 0;
 932
 933         switch (command) {
 934
 935         case SIOCSIFADDR:
 936 #ifdef INET
 937                 if (ifa->ifa_addr->sa_family == AF_INET)
 938                         avoid_reset = TRUE;
 939 #endif
 940 #ifdef INET6
 941                 if (ifa->ifa_addr->sa_family == AF_INET6)
 942                         avoid_reset = TRUE;
 943 #endif
 944 #if defined(INET) || defined(INET6)
 945                 /*
 946                 ** Calling init results in link renegotiation,
 947                 ** so we avoid doing it when possible.
 948                 */
 949                 if (avoid_reset) {
 950                         ifp->if_flags |= IFF_UP;
 951                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 952                                 ixgbe_init(adapter);
 953                         if (!(ifp->if_flags & IFF_NOARP))
 954                                 arp_ifinit(ifp, ifa);
 955                 } else
 956                         error = ether_ioctl(ifp, command, data);
 957 #endif
 958                 break;
 959         case SIOCSIFMTU:
 960                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 961                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 962                         error = EINVAL;
 963                 } else {
 964                         IXGBE_CORE_LOCK(adapter);
 965                         ifp->if_mtu = ifr->ifr_mtu;
 966                         adapter->max_frame_size =
 967                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 968                         ixgbe_init_locked(adapter);
 969                         IXGBE_CORE_UNLOCK(adapter);
 970                 }
 971                 break;
 972         case SIOCSIFFLAGS:
 973                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 974                 IXGBE_CORE_LOCK(adapter);
 975                 if (ifp->if_flags & IFF_UP) {
 976                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 977                                 if ((ifp->if_flags ^ adapter->if_flags) &
 978                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 979                                         ixgbe_set_promisc(adapter);
 980                                 }
 981                         } else
 982                                 ixgbe_init_locked(adapter);
 983                 } else
 984                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 985                                 ixgbe_stop(adapter);
 986                 adapter->if_flags = ifp->if_flags;
 987                 IXGBE_CORE_UNLOCK(adapter);
 988                 break;
 989         case SIOCADDMULTI:
 990         case SIOCDELMULTI:
 991                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 992                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 993                         IXGBE_CORE_LOCK(adapter);
 994                         ixgbe_disable_intr(adapter);
 995                         ixgbe_set_multi(adapter);
 996                         ixgbe_enable_intr(adapter);
 997                         IXGBE_CORE_UNLOCK(adapter);
 998                 }
 999                 break;
1000         case SIOCSIFMEDIA:
1001         case SIOCGIFMEDIA:
1002                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1003                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1004                 break;
1005         case SIOCSIFCAP:
1006         {
1007                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1008                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1009                 if (mask & IFCAP_HWCSUM)
1010                         ifp->if_capenable ^= IFCAP_HWCSUM;
1011                 if (mask & IFCAP_TSO4)
1012                         ifp->if_capenable ^= IFCAP_TSO4;
1013                 if (mask & IFCAP_TSO6)
1014                         ifp->if_capenable ^= IFCAP_TSO6;
1015                 if (mask & IFCAP_LRO)
1016                         ifp->if_capenable ^= IFCAP_LRO;
1017                 if (mask & IFCAP_VLAN_HWTAGGING)
1018                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1019                 if (mask & IFCAP_VLAN_HWFILTER)
1020                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1021                 if (mask & IFCAP_VLAN_HWTSO)
1022                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1023                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1024                         IXGBE_CORE_LOCK(adapter);
1025                         ixgbe_init_locked(adapter);
1026                         IXGBE_CORE_UNLOCK(adapter);
1027                 }
1028                 VLAN_CAPABILITIES(ifp);
1029                 break;
1030         }
1031         case SIOCGI2C:
1032         {
1033                 struct ixgbe_i2c_req    i2c;
1034                 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1035                 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1036                 if (error)
1037                         break;
1038                 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1039                         error = EINVAL;
1040                         break;
1041                 }
1042                 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1043                     i2c.dev_addr, i2c.data);
1044                 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1045                 break;
1046         }
1047         default:
1048                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1049                 error = ether_ioctl(ifp, command, data);
1050                 break;
1051         }
1052
1053         return (error);
1054 }
1055
1056 /*********************************************************************
1057  *  Init entry point
1058  *
1059  *  This routine is used in two ways. It is used by the stack as
1060  *  init entry point in network interface structure. It is also used
1061  *  by the driver as a hw/sw initialization routine to get to a
1062  *  consistent state.
1063  *
1064  *  return 0 on success, positive on failure
1065  **********************************************************************/
1066 #define IXGBE_MHADD_MFS_SHIFT 16
1067
1068 static void
1069 ixgbe_init_locked(struct adapter *adapter)
1070 {
1071         struct ifnet   *ifp = adapter->ifp;
1072         device_t        dev = adapter->dev;
1073         struct ixgbe_hw *hw = &adapter->hw;
1074         u32             k, txdctl, mhadd, gpie;
1075         u32             rxdctl, rxctrl;
1076
1077         mtx_assert(&adapter->core_mtx, MA_OWNED);
1078         INIT_DEBUGOUT("ixgbe_init: begin");
1079         hw->adapter_stopped = FALSE;
1080         ixgbe_stop_adapter(hw);
1081         callout_stop(&adapter->timer);
1082
1083         /* reprogram the RAR[0] in case user changed it. */
1084         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1085
1086         /* Get the latest mac address, User can use a LAA */
1087         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1088               IXGBE_ETH_LENGTH_OF_ADDRESS);
1089         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1090         hw->addr_ctrl.rar_used_count = 1;
1091
1092         /* Set the various hardware offload abilities */
1093         ifp->if_hwassist = 0;
1094         if (ifp->if_capenable & IFCAP_TSO)
1095                 ifp->if_hwassist |= CSUM_TSO;
1096         if (ifp->if_capenable & IFCAP_TXCSUM) {
1097                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1098 #if __FreeBSD_version >= 800000
1099                 if (hw->mac.type != ixgbe_mac_82598EB)
1100                         ifp->if_hwassist |= CSUM_SCTP;
1101 #endif
1102         }
1103
1104         /* Prepare transmit descriptors and buffers */
1105         if (ixgbe_setup_transmit_structures(adapter)) {
1106                 device_printf(dev,"Could not setup transmit structures\n");
1107                 ixgbe_stop(adapter);
1108                 return;
1109         }
1110
1111         ixgbe_init_hw(hw);
1112         ixgbe_initialize_transmit_units(adapter);
1113
1114         /* Setup Multicast table */
1115         ixgbe_set_multi(adapter);
1116
1117         /*
1118         ** Determine the correct mbuf pool
1119         ** for doing jumbo frames
1120         */
1121         if (adapter->max_frame_size <= 2048)
1122                 adapter->rx_mbuf_sz = MCLBYTES;
1123         else if (adapter->max_frame_size <= 4096)
1124                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1125         else if (adapter->max_frame_size <= 9216)
1126                 adapter->rx_mbuf_sz = MJUM9BYTES;
1127         else
1128                 adapter->rx_mbuf_sz = MJUM16BYTES;
1129
1130         /* Prepare receive descriptors and buffers */
1131         if (ixgbe_setup_receive_structures(adapter)) {
1132                 device_printf(dev,"Could not setup receive structures\n");
1133                 ixgbe_stop(adapter);
1134                 return;
1135         }
1136
1137         /* Configure RX settings */
1138         ixgbe_initialize_receive_units(adapter);
1139
1140         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1141
1142         /* Enable Fan Failure Interrupt */
1143         gpie |= IXGBE_SDP1_GPIEN;
1144
1145         /* Add for Module detection */
1146         if (hw->mac.type == ixgbe_mac_82599EB)
1147                 gpie |= IXGBE_SDP2_GPIEN;
1148
1149         /* Thermal Failure Detection */
1150         if (hw->mac.type == ixgbe_mac_X540)
1151                 gpie |= IXGBE_SDP0_GPIEN;
1152
1153         if (adapter->msix > 1) {
1154                 /* Enable Enhanced MSIX mode */
1155                 gpie |= IXGBE_GPIE_MSIX_MODE;
1156                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1157                     IXGBE_GPIE_OCD;
1158         }
1159         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1160
1161         /* Set MTU size */
1162         if (ifp->if_mtu > ETHERMTU) {
1163                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1164                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1165                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1166                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1167         }
1168
1169         /* Now enable all the queues */
1170
1171         for (int i = 0; i < adapter->num_queues; i++) {
1172                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1173                 txdctl |= IXGBE_TXDCTL_ENABLE;
1174                 /* Set WTHRESH to 8, burst writeback */
1175                 txdctl |= (8 << 16);
1176                 /*
1177                  * When the internal queue falls below PTHRESH (32),
1178                  * start prefetching as long as there are at least
1179                  * HTHRESH (1) buffers ready. The values are taken
1180                  * from the Intel linux driver 3.8.21.
1181                  * Prefetching enables tx line rate even with 1 queue.
1182                  */
1183                 txdctl |= (32 << 0) | (1 << 8);
1184                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1185         }
1186
1187         for (int i = 0; i < adapter->num_queues; i++) {
1188                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1189                 if (hw->mac.type == ixgbe_mac_82598EB) {
1190                         /*
1191                         ** PTHRESH = 21
1192                         ** HTHRESH = 4
1193                         ** WTHRESH = 8
1194                         */
1195                         rxdctl &= ~0x3FFFFF;
1196                         rxdctl |= 0x080420;
1197                 }
1198                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1199                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1200                 for (k = 0; k < 10; k++) {
1201                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1202                             IXGBE_RXDCTL_ENABLE)
1203                                 break;
1204                         else
1205                                 msec_delay(1);
1206                 }
1207                 wmb();
1208 #ifdef DEV_NETMAP
1209                 /*
1210                  * In netmap mode, we must preserve the buffers made
1211                  * available to userspace before the if_init()
1212                  * (this is true by default on the TX side, because
1213                  * init makes all buffers available to userspace).
1214                  *
1215                  * netmap_reset() and the device specific routines
1216                  * (e.g. ixgbe_setup_receive_rings()) map these
1217                  * buffers at the end of the NIC ring, so here we
1218                  * must set the RDT (tail) register to make sure
1219                  * they are not overwritten.
1220                  *
1221                  * In this driver the NIC ring starts at RDH = 0,
1222                  * RDT points to the last slot available for reception (?),
1223                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1224                  */
1225                 if (ifp->if_capenable & IFCAP_NETMAP) {
1226                         struct netmap_adapter *na = NA(adapter->ifp);
1227                         struct netmap_kring *kring = &na->rx_rings[i];
1228                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1229
1230                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1231                 } else
1232 #endif /* DEV_NETMAP */
1233                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1234         }
1235
1236         /* Set up VLAN support and filter */
1237         ixgbe_setup_vlan_hw_support(adapter);
1238
1239         /* Enable Receive engine */
1240         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1241         if (hw->mac.type == ixgbe_mac_82598EB)
1242                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1243         rxctrl |= IXGBE_RXCTRL_RXEN;
1244         ixgbe_enable_rx_dma(hw, rxctrl);
1245
1246         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1247
1248         /* Set up MSI/X routing */
1249         if (ixgbe_enable_msix)  {
1250                 ixgbe_configure_ivars(adapter);
1251                 /* Set up auto-mask */
1252                 if (hw->mac.type == ixgbe_mac_82598EB)
1253                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1254                 else {
1255                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1256                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1257                 }
1258         } else {  /* Simple settings for Legacy/MSI */
1259                 ixgbe_set_ivar(adapter, 0, 0, 0);
1260                 ixgbe_set_ivar(adapter, 0, 0, 1);
1261                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1262         }
1263
1264 #ifdef IXGBE_FDIR
1265         /* Init Flow director */
1266         if (hw->mac.type != ixgbe_mac_82598EB) {
1267                 u32 hdrm = 32 << fdir_pballoc;
1268
1269                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1270                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1271         }
1272 #endif
1273
1274         /*
1275         ** Check on any SFP devices that
1276         ** need to be kick-started
1277         */
1278         if (hw->phy.type == ixgbe_phy_none) {
1279                 int err = hw->phy.ops.identify(hw);
1280                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1281                         device_printf(dev,
1282                             "Unsupported SFP+ module type was detected.\n");
1283                         return;
1284                 }
1285         }
1286
1287         /* Set moderation on the Link interrupt */
1288         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1289
1290         /* Config/Enable Link */
1291         ixgbe_config_link(adapter);
1292
1293         /* Hardware Packet Buffer & Flow Control setup */
1294         {
1295                 u32 rxpb, frame, size, tmp;
1296
1297                 frame = adapter->max_frame_size;
1298
1299                 /* Calculate High Water */
1300                 if (hw->mac.type == ixgbe_mac_X540)
1301                         tmp = IXGBE_DV_X540(frame, frame);
1302                 else
1303                         tmp = IXGBE_DV(frame, frame);
1304                 size = IXGBE_BT2KB(tmp);
1305                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1306                 hw->fc.high_water[0] = rxpb - size;
1307
1308                 /* Now calculate Low Water */
1309                 if (hw->mac.type == ixgbe_mac_X540)
1310                         tmp = IXGBE_LOW_DV_X540(frame);
1311                 else
1312                         tmp = IXGBE_LOW_DV(frame);
1313                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1314
1315                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1316                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1317                 hw->fc.send_xon = TRUE;
1318         }
1319         /* Initialize the FC settings */
1320         ixgbe_start_hw(hw);
1321
1322         /* And now turn on interrupts */
1323         ixgbe_enable_intr(adapter);
1324
1325         /* Now inform the stack we're ready */
1326         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1327
1328         return;
1329 }
1330
1331 static void
1332 ixgbe_init(void *arg)
1333 {
1334         struct adapter *adapter = arg;
1335
1336         IXGBE_CORE_LOCK(adapter);
1337         ixgbe_init_locked(adapter);
1338         IXGBE_CORE_UNLOCK(adapter);
1339         return;
1340 }
1341
1342
1343 /*
1344 **
1345 ** MSIX Interrupt Handlers and Tasklets
1346 **
1347 */
1348
1349 static inline void
1350 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1351 {
1352         struct ixgbe_hw *hw = &adapter->hw;
1353         u64     queue = (u64)(1 << vector);
1354         u32     mask;
1355
1356         if (hw->mac.type == ixgbe_mac_82598EB) {
1357                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1358                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1359         } else {
1360                 mask = (queue & 0xFFFFFFFF);
1361                 if (mask)
1362                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1363                 mask = (queue >> 32);
1364                 if (mask)
1365                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1366         }
1367 }
1368
1369 static inline void
1370 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1371 {
1372         struct ixgbe_hw *hw = &adapter->hw;
1373         u64     queue = (u64)(1 << vector);
1374         u32     mask;
1375
1376         if (hw->mac.type == ixgbe_mac_82598EB) {
1377                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1379         } else {
1380                 mask = (queue & 0xFFFFFFFF);
1381                 if (mask)
1382                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1383                 mask = (queue >> 32);
1384                 if (mask)
1385                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1386         }
1387 }
1388
1389 static inline void
1390 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1391 {
1392         u32 mask;
1393
1394         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1395                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1396                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1397         } else {
1398                 mask = (queues & 0xFFFFFFFF);
1399                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1400                 mask = (queues >> 32);
1401                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1402         }
1403 }
1404
1405
1406 static void
1407 ixgbe_handle_que(void *context, int pending)
1408 {
1409         struct ix_queue *que = context;
1410         struct adapter  *adapter = que->adapter;
1411         struct tx_ring  *txr = que->txr;
1412         struct ifnet    *ifp = adapter->ifp;
1413         bool            more;
1414
1415         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1416                 more = ixgbe_rxeof(que);
1417                 IXGBE_TX_LOCK(txr);
1418                 ixgbe_txeof(txr);
1419 #ifndef IXGBE_LEGACY_TX
1420                 if (!drbr_empty(ifp, txr->br))
1421                         ixgbe_mq_start_locked(ifp, txr, NULL);
1422 #else
1423                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1424                         ixgbe_start_locked(txr, ifp);
1425 #endif
1426                 IXGBE_TX_UNLOCK(txr);
1427                 if (more) {
1428                         taskqueue_enqueue(que->tq, &que->que_task);
1429                         return;
1430                 }
1431         }
1432
1433         /* Reenable this interrupt */
1434         ixgbe_enable_queue(adapter, que->msix);
1435         return;
1436 }
1437
1438
1439 /*********************************************************************
1440  *
1441  *  Legacy Interrupt Service routine
1442  *
1443  **********************************************************************/
1444
1445 static void
1446 ixgbe_legacy_irq(void *arg)
1447 {
1448         struct ix_queue *que = arg;
1449         struct adapter  *adapter = que->adapter;
1450         struct ixgbe_hw *hw = &adapter->hw;
1451         struct          tx_ring *txr = adapter->tx_rings;
1452         bool            more_tx, more_rx;
1453         u32             reg_eicr, loop = MAX_LOOP;
1454
1455
1456         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1457
1458         ++que->irqs;
1459         if (reg_eicr == 0) {
1460                 ixgbe_enable_intr(adapter);
1461                 return;
1462         }
1463
1464         more_rx = ixgbe_rxeof(que);
1465
1466         IXGBE_TX_LOCK(txr);
1467         do {
1468                 more_tx = ixgbe_txeof(txr);
1469         } while (loop-- && more_tx);
1470         IXGBE_TX_UNLOCK(txr);
1471
1472         if (more_rx || more_tx)
1473                 taskqueue_enqueue(que->tq, &que->que_task);
1474
1475         /* Check for fan failure */
1476         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1477             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1478                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1479                     "REPLACE IMMEDIATELY!!\n");
1480                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1481         }
1482
1483         /* Link status change */
1484         if (reg_eicr & IXGBE_EICR_LSC)
1485                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1486
1487         ixgbe_enable_intr(adapter);
1488         return;
1489 }
1490
1491
1492 /*********************************************************************
1493  *
1494  *  MSIX Queue Interrupt Service routine
1495  *
1496  **********************************************************************/
1497 void
1498 ixgbe_msix_que(void *arg)
1499 {
1500         struct ix_queue *que = arg;
1501         struct adapter  *adapter = que->adapter;
1502         struct tx_ring  *txr = que->txr;
1503         struct rx_ring  *rxr = que->rxr;
1504         bool            more_tx, more_rx;
1505         u32             newitr = 0;
1506
1507         ixgbe_disable_queue(adapter, que->msix);
1508         ++que->irqs;
1509
1510         more_rx = ixgbe_rxeof(que);
1511
1512         IXGBE_TX_LOCK(txr);
1513         more_tx = ixgbe_txeof(txr);
1514         /*
1515         ** Make certain that if the stack
1516         ** has anything queued the task gets
1517         ** scheduled to handle it.
1518         */
1519 #ifdef IXGBE_LEGACY_TX
1520         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1521 #else
1522         if (!drbr_empty(adapter->ifp, txr->br))
1523 #endif
1524                 more_tx = 1;
1525         IXGBE_TX_UNLOCK(txr);
1526
1527         /* Do AIM now? */
1528
1529         if (ixgbe_enable_aim == FALSE)
1530                 goto no_calc;
1531         /*
1532         ** Do Adaptive Interrupt Moderation:
1533         **  - Write out last calculated setting
1534         **  - Calculate based on average size over
1535         **    the last interval.
1536         */
1537         if (que->eitr_setting)
1538                 IXGBE_WRITE_REG(&adapter->hw,
1539                     IXGBE_EITR(que->msix), que->eitr_setting);
1540
1541         que->eitr_setting = 0;
1542
1543         /* Idle, do nothing */
1544         if ((txr->bytes == 0) && (rxr->bytes == 0))
1545                 goto no_calc;
1546
1547         if ((txr->bytes) && (txr->packets))
1548                 newitr = txr->bytes/txr->packets;
1549         if ((rxr->bytes) && (rxr->packets))
1550                 newitr = max(newitr,
1551                     (rxr->bytes / rxr->packets));
1552         newitr += 24; /* account for hardware frame, crc */
1553
1554         /* set an upper boundary */
1555         newitr = min(newitr, 3000);
1556
1557         /* Be nice to the mid range */
1558         if ((newitr > 300) && (newitr < 1200))
1559                 newitr = (newitr / 3);
1560         else
1561                 newitr = (newitr / 2);
1562
1563         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1564                 newitr |= newitr << 16;
1565         else
1566                 newitr |= IXGBE_EITR_CNT_WDIS;
1567
1568         /* save for next interrupt */
1569         que->eitr_setting = newitr;
1570
1571         /* Reset state */
1572         txr->bytes = 0;
1573         txr->packets = 0;
1574         rxr->bytes = 0;
1575         rxr->packets = 0;
1576
1577 no_calc:
1578         if (more_tx || more_rx)
1579                 taskqueue_enqueue(que->tq, &que->que_task);
1580         else /* Reenable this interrupt */
1581                 ixgbe_enable_queue(adapter, que->msix);
1582         return;
1583 }
1584
1585
1586 static void
1587 ixgbe_msix_link(void *arg)
1588 {
1589         struct adapter  *adapter = arg;
1590         struct ixgbe_hw *hw = &adapter->hw;
1591         u32             reg_eicr;
1592
1593         ++adapter->link_irq;
1594
1595         /* First get the cause */
1596         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1597         /* Clear interrupt with write */
1598         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1599
1600         /* Link status change */
1601         if (reg_eicr & IXGBE_EICR_LSC)
1602                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1603
1604         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1605 #ifdef IXGBE_FDIR
1606                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1607                         /* This is probably overkill :) */
1608                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1609                                 return;
1610                         /* Disable the interrupt */
1611                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1612                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1613                 } else
1614 #endif
1615                 if (reg_eicr & IXGBE_EICR_ECC) {
1616                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1617                             "Please Reboot!!\n");
1618                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1619                 } else
1620
1621                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1622                         /* Clear the interrupt */
1623                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1624                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1625                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1626                         /* Clear the interrupt */
1627                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1628                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1629                 }
1630         }
1631
1632         /* Check for fan failure */
1633         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1634             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1635                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1636                     "REPLACE IMMEDIATELY!!\n");
1637                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1638         }
1639
1640         /* Check for over temp condition */
1641         if ((hw->mac.type == ixgbe_mac_X540) &&
1642             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1643                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1644                     "PHY IS SHUT DOWN!!\n");
1645                 device_printf(adapter->dev, "System shutdown required\n");
1646                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1647         }
1648
1649         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1650         return;
1651 }
1652
1653 /*********************************************************************
1654  *
1655  *  Media Ioctl callback
1656  *
1657  *  This routine is called whenever the user queries the status of
1658  *  the interface using ifconfig.
1659  *
1660  **********************************************************************/
1661 static void
1662 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1663 {
1664         struct adapter *adapter = ifp->if_softc;
1665
1666         INIT_DEBUGOUT("ixgbe_media_status: begin");
1667         IXGBE_CORE_LOCK(adapter);
1668         ixgbe_update_link_status(adapter);
1669
1670         ifmr->ifm_status = IFM_AVALID;
1671         ifmr->ifm_active = IFM_ETHER;
1672
1673         if (!adapter->link_active) {
1674                 IXGBE_CORE_UNLOCK(adapter);
1675                 return;
1676         }
1677
1678         ifmr->ifm_status |= IFM_ACTIVE;
1679
1680         switch (adapter->link_speed) {
1681                 case IXGBE_LINK_SPEED_100_FULL:
1682                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1683                         break;
1684                 case IXGBE_LINK_SPEED_1GB_FULL:
1685                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1686                         break;
1687                 case IXGBE_LINK_SPEED_10GB_FULL:
1688                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1689                         break;
1690         }
1691
1692         IXGBE_CORE_UNLOCK(adapter);
1693
1694         return;
1695 }
1696
1697 /*********************************************************************
1698  *
1699  *  Media Ioctl callback
1700  *
1701  *  This routine is called when the user changes speed/duplex using
1702  *  media/mediopt option with ifconfig.
1703  *
1704  **********************************************************************/
1705 static int
1706 ixgbe_media_change(struct ifnet * ifp)
1707 {
1708         struct adapter *adapter = ifp->if_softc;
1709         struct ifmedia *ifm = &adapter->media;
1710
1711         INIT_DEBUGOUT("ixgbe_media_change: begin");
1712
1713         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1714                 return (EINVAL);
1715
1716         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1717         case IFM_AUTO:
1718                 adapter->hw.phy.autoneg_advertised =
1719                     IXGBE_LINK_SPEED_100_FULL |
1720                     IXGBE_LINK_SPEED_1GB_FULL |
1721                     IXGBE_LINK_SPEED_10GB_FULL;
1722                 break;
1723         default:
1724                 device_printf(adapter->dev, "Only auto media type\n");
1725                 return (EINVAL);
1726         }
1727
1728         return (0);
1729 }
1730
1731 /*********************************************************************
1732  *
1733  *  This routine maps the mbufs to tx descriptors, allowing the
1734  *  TX engine to transmit the packets.
1735  *      - return 0 on success, positive on failure
1736  *
1737  **********************************************************************/
1738
1739 static int
1740 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1741 {
1742         struct adapter  *adapter = txr->adapter;
1743         u32             olinfo_status = 0, cmd_type_len;
1744         int             i, j, error, nsegs;
1745         int             first;
1746         bool            remap = TRUE;
1747         struct mbuf     *m_head;
1748         bus_dma_segment_t segs[adapter->num_segs];
1749         bus_dmamap_t    map;
1750         struct ixgbe_tx_buf *txbuf;
1751         union ixgbe_adv_tx_desc *txd = NULL;
1752
1753         m_head = *m_headp;
1754
1755         /* Basic descriptor defines */
1756         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1757             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1758
1759         if (m_head->m_flags & M_VLANTAG)
1760                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1761
1762         /*
1763          * Important to capture the first descriptor
1764          * used because it will contain the index of
1765          * the one we tell the hardware to report back
1766          */
1767         first = txr->next_avail_desc;
1768         txbuf = &txr->tx_buffers[first];
1769         map = txbuf->map;
1770
1771         /*
1772          * Map the packet for DMA.
1773          */
1774 retry:
1775         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1776             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1777
1778         if (__predict_false(error)) {
1779                 struct mbuf *m;
1780
1781                 switch (error) {
1782                 case EFBIG:
1783                         /* Try it again? - one try */
1784                         if (remap == TRUE) {
1785                                 remap = FALSE;
1786                                 m = m_defrag(*m_headp, M_NOWAIT);
1787                                 if (m == NULL) {
1788                                         adapter->mbuf_defrag_failed++;
1789                                         m_freem(*m_headp);
1790                                         *m_headp = NULL;
1791                                         return (ENOBUFS);
1792                                 }
1793                                 *m_headp = m;
1794                                 goto retry;
1795                         } else
1796                                 return (error);
1797                 case ENOMEM:
1798                         txr->no_tx_dma_setup++;
1799                         return (error);
1800                 default:
1801                         txr->no_tx_dma_setup++;
1802                         m_freem(*m_headp);
1803                         *m_headp = NULL;
1804                         return (error);
1805                 }
1806         }
1807
1808         /* Make certain there are enough descriptors */
1809         if (nsegs > txr->tx_avail - 2) {
1810                 txr->no_desc_avail++;
1811                 bus_dmamap_unload(txr->txtag, map);
1812                 return (ENOBUFS);
1813         }
1814         m_head = *m_headp;
1815
1816         /*
1817         ** Set up the appropriate offload context
1818         ** this will consume the first descriptor
1819         */
1820         error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1821         if (__predict_false(error)) {
1822                 if (error == ENOBUFS)
1823                         *m_headp = NULL;
1824                 return (error);
1825         }
1826
1827 #ifdef IXGBE_FDIR
1828         /* Do the flow director magic */
1829         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1830                 ++txr->atr_count;
1831                 if (txr->atr_count >= atr_sample_rate) {
1832                         ixgbe_atr(txr, m_head);
1833                         txr->atr_count = 0;
1834                 }
1835         }
1836 #endif
1837
1838         i = txr->next_avail_desc;
1839         for (j = 0; j < nsegs; j++) {
1840                 bus_size_t seglen;
1841                 bus_addr_t segaddr;
1842
1843                 txbuf = &txr->tx_buffers[i];
1844                 txd = &txr->tx_base[i];
1845                 seglen = segs[j].ds_len;
1846                 segaddr = htole64(segs[j].ds_addr);
1847
1848                 txd->read.buffer_addr = segaddr;
1849                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1850                     cmd_type_len |seglen);
1851                 txd->read.olinfo_status = htole32(olinfo_status);
1852
1853                 if (++i == txr->num_desc)
1854                         i = 0;
1855         }
1856
1857         txd->read.cmd_type_len |=
1858             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1859         txr->tx_avail -= nsegs;
1860         txr->next_avail_desc = i;
1861
1862         txbuf->m_head = m_head;
1863         /*
1864         ** Here we swap the map so the last descriptor,
1865         ** which gets the completion interrupt has the
1866         ** real map, and the first descriptor gets the
1867         ** unused map from this descriptor.
1868         */
1869         txr->tx_buffers[first].map = txbuf->map;
1870         txbuf->map = map;
1871         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1872
1873         /* Set the EOP descriptor that will be marked done */
1874         txbuf = &txr->tx_buffers[first];
1875         txbuf->eop = txd;
1876
1877         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1878             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1879         /*
1880          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1881          * hardware that this frame is available to transmit.
1882          */
1883         ++txr->total_packets;
1884         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1885
1886         return (0);
1887
1888 }
1889
1890 static void
1891 ixgbe_set_promisc(struct adapter *adapter)
1892 {
1893         u_int32_t       reg_rctl;
1894         struct ifnet   *ifp = adapter->ifp;
1895
1896         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1897         reg_rctl &= (~IXGBE_FCTRL_UPE);
1898         reg_rctl &= (~IXGBE_FCTRL_MPE);
1899         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1900
1901         if (ifp->if_flags & IFF_PROMISC) {
1902                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1903                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1904         } else if (ifp->if_flags & IFF_ALLMULTI) {
1905                 reg_rctl |= IXGBE_FCTRL_MPE;
1906                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1907                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1908         }
1909         return;
1910 }
1911
1912
1913 /*********************************************************************
1914  *  Multicast Update
1915  *
1916  *  This routine is called whenever multicast address list is updated.
1917  *
1918  **********************************************************************/
1919 #define IXGBE_RAR_ENTRIES 16
1920
1921 static void
1922 ixgbe_set_multi(struct adapter *adapter)
1923 {
1924         u32     fctrl;
1925         u8      *mta;
1926         u8      *update_ptr;
1927         struct  ifmultiaddr *ifma;
1928         int     mcnt = 0;
1929         struct ifnet   *ifp = adapter->ifp;
1930
1931         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1932
1933         mta = adapter->mta;
1934         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1935             MAX_NUM_MULTICAST_ADDRESSES);
1936
1937         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1938         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1939         if (ifp->if_flags & IFF_PROMISC)
1940                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1941         else if (ifp->if_flags & IFF_ALLMULTI) {
1942                 fctrl |= IXGBE_FCTRL_MPE;
1943                 fctrl &= ~IXGBE_FCTRL_UPE;
1944         } else
1945                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1946
1947         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1948
1949 #if __FreeBSD_version < 800000
1950         IF_ADDR_LOCK(ifp);
1951 #else
1952         if_maddr_rlock(ifp);
1953 #endif
1954         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1955                 if (ifma->ifma_addr->sa_family != AF_LINK)
1956                         continue;
1957                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1958                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1959                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1960                 mcnt++;
1961         }
1962 #if __FreeBSD_version < 800000
1963         IF_ADDR_UNLOCK(ifp);
1964 #else
1965         if_maddr_runlock(ifp);
1966 #endif
1967
1968         update_ptr = mta;
1969         ixgbe_update_mc_addr_list(&adapter->hw,
1970             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1971
1972         return;
1973 }
1974
1975 /*
1976  * This is an iterator function now needed by the multicast
1977  * shared code. It simply feeds the shared code routine the
1978  * addresses in the array of ixgbe_set_multi() one by one.
1979  */
1980 static u8 *
1981 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1982 {
1983         u8 *addr = *update_ptr;
1984         u8 *newptr;
1985         *vmdq = 0;
1986
1987         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1988         *update_ptr = newptr;
1989         return addr;
1990 }
1991
1992
1993 /*********************************************************************
1994  *  Timer routine
1995  *
1996  *  This routine checks for link status,updates statistics,
1997  *  and runs the watchdog check.
1998  *
1999  **********************************************************************/
2000
2001 static void
2002 ixgbe_local_timer(void *arg)
2003 {
2004         struct adapter  *adapter = arg;
2005         device_t        dev = adapter->dev;
2006         struct ix_queue *que = adapter->queues;
2007         struct tx_ring  *txr = adapter->tx_rings;
2008         int             hung = 0, paused = 0;
2009
2010         mtx_assert(&adapter->core_mtx, MA_OWNED);
2011
2012         /* Check for pluggable optics */
2013         if (adapter->sfp_probe)
2014                 if (!ixgbe_sfp_probe(adapter))
2015                         goto out; /* Nothing to do */
2016
2017         ixgbe_update_link_status(adapter);
2018         ixgbe_update_stats_counters(adapter);
2019
2020         /*
2021          * If the interface has been paused
2022          * then don't do the watchdog check
2023          */
2024         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2025                 paused = 1;
2026
2027         /*
2028         ** Check the TX queues status
2029         **      - watchdog only if all queues show hung
2030         */
2031         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2032                 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2033                     (paused == 0))
2034                         ++hung;
2035                 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2036                         taskqueue_enqueue(que->tq, &que->que_task);
2037         }
2038         /* Only truely watchdog if all queues show hung */
2039         if (hung == adapter->num_queues)
2040                 goto watchdog;
2041
2042 out:
2043         ixgbe_rearm_queues(adapter, adapter->que_mask);
2044         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2045         return;
2046
2047 watchdog:
2048         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2049         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2050             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2051             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2052         device_printf(dev,"TX(%d) desc avail = %d,"
2053             "Next TX to Clean = %d\n",
2054             txr->me, txr->tx_avail, txr->next_to_clean);
2055         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2056         adapter->watchdog_events++;
2057         ixgbe_init_locked(adapter);
2058 }
2059
2060 /*
2061 ** Note: this routine updates the OS on the link state
2062 **      the real check of the hardware only happens with
2063 **      a link interrupt.
2064 */
2065 static void
2066 ixgbe_update_link_status(struct adapter *adapter)
2067 {
2068         struct ifnet    *ifp = adapter->ifp;
2069         device_t dev = adapter->dev;
2070
2071
2072         if (adapter->link_up){
2073                 if (adapter->link_active == FALSE) {
2074                         if (bootverbose)
2075                                 device_printf(dev,"Link is up %d Gbps %s \n",
2076                                     ((adapter->link_speed == 128)? 10:1),
2077                                     "Full Duplex");
2078                         adapter->link_active = TRUE;
2079                         /* Update any Flow Control changes */
2080                         ixgbe_fc_enable(&adapter->hw);
2081                         if_link_state_change(ifp, LINK_STATE_UP);
2082                 }
2083         } else { /* Link down */
2084                 if (adapter->link_active == TRUE) {
2085                         if (bootverbose)
2086                                 device_printf(dev,"Link is Down\n");
2087                         if_link_state_change(ifp, LINK_STATE_DOWN);
2088                         adapter->link_active = FALSE;
2089                 }
2090         }
2091
2092         return;
2093 }
2094
2095
2096 /*********************************************************************
2097  *
2098  *  This routine disables all traffic on the adapter by issuing a
2099  *  global reset on the MAC and deallocates TX/RX buffers.
2100  *
2101  **********************************************************************/
2102
2103 static void
2104 ixgbe_stop(void *arg)
2105 {
2106         struct ifnet   *ifp;
2107         struct adapter *adapter = arg;
2108         struct ixgbe_hw *hw = &adapter->hw;
2109         ifp = adapter->ifp;
2110
2111         mtx_assert(&adapter->core_mtx, MA_OWNED);
2112
2113         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2114         ixgbe_disable_intr(adapter);
2115         callout_stop(&adapter->timer);
2116
2117         /* Let the stack know...*/
2118         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2119
2120         ixgbe_reset_hw(hw);
2121         hw->adapter_stopped = FALSE;
2122         ixgbe_stop_adapter(hw);
2123         /* Turn off the laser */
2124         if (hw->phy.multispeed_fiber)
2125                 ixgbe_disable_tx_laser(hw);
2126
2127         /* reprogram the RAR[0] in case user changed it. */
2128         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2129
2130         return;
2131 }
2132
2133
2134 /*********************************************************************
2135  *
2136  *  Determine hardware revision.
2137  *
2138  **********************************************************************/
2139 static void
2140 ixgbe_identify_hardware(struct adapter *adapter)
2141 {
2142         device_t        dev = adapter->dev;
2143         struct ixgbe_hw *hw = &adapter->hw;
2144
2145         /* Save off the information about this board */
2146         hw->vendor_id = pci_get_vendor(dev);
2147         hw->device_id = pci_get_device(dev);
2148         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2149         hw->subsystem_vendor_id =
2150             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2151         hw->subsystem_device_id =
2152             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2153
2154         /* We need this here to set the num_segs below */
2155         ixgbe_set_mac_type(hw);
2156
2157         /* Pick up the 82599 and VF settings */
2158         if (hw->mac.type != ixgbe_mac_82598EB) {
2159                 hw->phy.smart_speed = ixgbe_smart_speed;
2160                 adapter->num_segs = IXGBE_82599_SCATTER;
2161         } else
2162                 adapter->num_segs = IXGBE_82598_SCATTER;
2163
2164         return;
2165 }
2166
2167 /*********************************************************************
2168  *
2169  *  Determine optic type
2170  *
2171  **********************************************************************/
2172 static void
2173 ixgbe_setup_optics(struct adapter *adapter)
2174 {
2175         struct ixgbe_hw *hw = &adapter->hw;
2176         int             layer;
2177
2178         layer = ixgbe_get_supported_physical_layer(hw);
2179
2180         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2181                 adapter->optics = IFM_10G_T;
2182                 return;
2183         }
2184
2185         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2186                 adapter->optics = IFM_1000_T;
2187                 return;
2188         }
2189
2190         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2191                 adapter->optics = IFM_1000_SX;
2192                 return;
2193         }
2194
2195         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2196             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2197                 adapter->optics = IFM_10G_LR;
2198                 return;
2199         }
2200
2201         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2202                 adapter->optics = IFM_10G_SR;
2203                 return;
2204         }
2205
2206         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2207                 adapter->optics = IFM_10G_TWINAX;
2208                 return;
2209         }
2210
2211         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2212             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2213                 adapter->optics = IFM_10G_CX4;
2214                 return;
2215         }
2216
2217         /* If we get here just set the default */
2218         adapter->optics = IFM_ETHER | IFM_AUTO;
2219         return;
2220 }
2221
2222 /*********************************************************************
2223  *
2224  *  Setup the Legacy or MSI Interrupt handler
2225  *
2226  **********************************************************************/
2227 static int
2228 ixgbe_allocate_legacy(struct adapter *adapter)
2229 {
2230         device_t        dev = adapter->dev;
2231         struct          ix_queue *que = adapter->queues;
2232 #ifndef IXGBE_LEGACY_TX
2233         struct tx_ring          *txr = adapter->tx_rings;
2234 #endif
2235         int             error, rid = 0;
2236
2237         /* MSI RID at 1 */
2238         if (adapter->msix == 1)
2239                 rid = 1;
2240
2241         /* We allocate a single interrupt resource */
2242         adapter->res = bus_alloc_resource_any(dev,
2243             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2244         if (adapter->res == NULL) {
2245                 device_printf(dev, "Unable to allocate bus resource: "
2246                     "interrupt\n");
2247                 return (ENXIO);
2248         }
2249
2250         /*
2251          * Try allocating a fast interrupt and the associated deferred
2252          * processing contexts.
2253          */
2254 #ifndef IXGBE_LEGACY_TX
2255         TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2256 #endif
2257         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2258         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2259             taskqueue_thread_enqueue, &que->tq);
2260         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2261             device_get_nameunit(adapter->dev));
2262
2263         /* Tasklets for Link, SFP and Multispeed Fiber */
2264         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2265         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2266         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2267 #ifdef IXGBE_FDIR
2268         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2269 #endif
2270         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2271             taskqueue_thread_enqueue, &adapter->tq);
2272         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2273             device_get_nameunit(adapter->dev));
2274
2275         if ((error = bus_setup_intr(dev, adapter->res,
2276             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2277             que, &adapter->tag)) != 0) {
2278                 device_printf(dev, "Failed to register fast interrupt "
2279                     "handler: %d\n", error);
2280                 taskqueue_free(que->tq);
2281                 taskqueue_free(adapter->tq);
2282                 que->tq = NULL;
2283                 adapter->tq = NULL;
2284                 return (error);
2285         }
2286         /* For simplicity in the handlers */
2287         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2288
2289         return (0);
2290 }
2291
2292
2293 /*********************************************************************
2294  *
2295  *  Setup MSIX Interrupt resources and handlers
2296  *
2297  **********************************************************************/
2298 static int
2299 ixgbe_allocate_msix(struct adapter *adapter)
2300 {
2301         device_t        dev = adapter->dev;
2302         struct          ix_queue *que = adapter->queues;
2303         struct          tx_ring *txr = adapter->tx_rings;
2304         int             error, rid, vector = 0;
2305
2306         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2307                 rid = vector + 1;
2308                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2309                     RF_SHAREABLE | RF_ACTIVE);
2310                 if (que->res == NULL) {
2311                         device_printf(dev,"Unable to allocate"
2312                             " bus resource: que interrupt [%d]\n", vector);
2313                         return (ENXIO);
2314                 }
2315                 /* Set the handler function */
2316                 error = bus_setup_intr(dev, que->res,
2317                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2318                     ixgbe_msix_que, que, &que->tag);
2319                 if (error) {
2320                         que->res = NULL;
2321                         device_printf(dev, "Failed to register QUE handler");
2322                         return (error);
2323                 }
2324 #if __FreeBSD_version >= 800504
2325                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2326 #endif
2327                 que->msix = vector;
2328                 adapter->que_mask |= (u64)(1 << que->msix);
2329                 /*
2330                 ** Bind the msix vector, and thus the
2331                 ** ring to the corresponding cpu.
2332                 */
2333                 if (adapter->num_queues > 1)
2334                         bus_bind_intr(dev, que->res, i);
2335
2336 #ifndef IXGBE_LEGACY_TX
2337                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2338 #endif
2339                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2340                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2341                     taskqueue_thread_enqueue, &que->tq);
2342                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2343                     device_get_nameunit(adapter->dev));
2344         }
2345
2346         /* and Link */
2347         rid = vector + 1;
2348         adapter->res = bus_alloc_resource_any(dev,
2349             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2350         if (!adapter->res) {
2351                 device_printf(dev,"Unable to allocate"
2352             " bus resource: Link interrupt [%d]\n", rid);
2353                 return (ENXIO);
2354         }
2355         /* Set the link handler function */
2356         error = bus_setup_intr(dev, adapter->res,
2357             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2358             ixgbe_msix_link, adapter, &adapter->tag);
2359         if (error) {
2360                 adapter->res = NULL;
2361                 device_printf(dev, "Failed to register LINK handler");
2362                 return (error);
2363         }
2364 #if __FreeBSD_version >= 800504
2365         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2366 #endif
2367         adapter->linkvec = vector;
2368         /* Tasklets for Link, SFP and Multispeed Fiber */
2369         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2370         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2371         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2372 #ifdef IXGBE_FDIR
2373         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2374 #endif
2375         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2376             taskqueue_thread_enqueue, &adapter->tq);
2377         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2378             device_get_nameunit(adapter->dev));
2379
2380         return (0);
2381 }
2382
2383 /*
2384  * Setup Either MSI/X or MSI
2385  */
2386 static int
2387 ixgbe_setup_msix(struct adapter *adapter)
2388 {
2389         device_t dev = adapter->dev;
2390         int rid, want, queues, msgs;
2391
2392         /* Override by tuneable */
2393         if (ixgbe_enable_msix == 0)
2394                 goto msi;
2395
2396         /* First try MSI/X */
2397         rid = PCIR_BAR(MSIX_82598_BAR);
2398         adapter->msix_mem = bus_alloc_resource_any(dev,
2399             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2400         if (!adapter->msix_mem) {
2401                 rid += 4;       /* 82599 maps in higher BAR */
2402                 adapter->msix_mem = bus_alloc_resource_any(dev,
2403                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2404         }
2405         if (!adapter->msix_mem) {
2406                 /* May not be enabled */
2407                 device_printf(adapter->dev,
2408                     "Unable to map MSIX table \n");
2409                 goto msi;
2410         }
2411
2412         msgs = pci_msix_count(dev);
2413         if (msgs == 0) { /* system has msix disabled */
2414                 bus_release_resource(dev, SYS_RES_MEMORY,
2415                     rid, adapter->msix_mem);
2416                 adapter->msix_mem = NULL;
2417                 goto msi;
2418         }
2419
2420         /* Figure out a reasonable auto config value */
2421         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2422
2423         if (ixgbe_num_queues != 0)
2424                 queues = ixgbe_num_queues;
2425         /* Set max queues to 8 when autoconfiguring */
2426         else if ((ixgbe_num_queues == 0) && (queues > 8))
2427                 queues = 8;
2428
2429         /*
2430         ** Want one vector (RX/TX pair) per queue
2431         ** plus an additional for Link.
2432         */
2433         want = queues + 1;
2434         if (msgs >= want)
2435                 msgs = want;
2436         else {
2437                 device_printf(adapter->dev,
2438                     "MSIX Configuration Problem, "
2439                     "%d vectors but %d queues wanted!\n",
2440                     msgs, want);
2441                 return (0); /* Will go to Legacy setup */
2442         }
2443         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2444                 device_printf(adapter->dev,
2445                     "Using MSIX interrupts with %d vectors\n", msgs);
2446                 adapter->num_queues = queues;
2447                 return (msgs);
2448         }
2449 msi:
2450         msgs = pci_msi_count(dev);
2451         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2452                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2453         else
2454                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2455         return (msgs);
2456 }
2457
2458
2459 static int
2460 ixgbe_allocate_pci_resources(struct adapter *adapter)
2461 {
2462         int             rid;
2463         device_t        dev = adapter->dev;
2464
2465         rid = PCIR_BAR(0);
2466         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2467             &rid, RF_ACTIVE);
2468
2469         if (!(adapter->pci_mem)) {
2470                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2471                 return (ENXIO);
2472         }
2473
2474         adapter->osdep.mem_bus_space_tag =
2475                 rman_get_bustag(adapter->pci_mem);
2476         adapter->osdep.mem_bus_space_handle =
2477                 rman_get_bushandle(adapter->pci_mem);
2478         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2479
2480         /* Legacy defaults */
2481         adapter->num_queues = 1;
2482         adapter->hw.back = &adapter->osdep;
2483
2484         /*
2485         ** Now setup MSI or MSI/X, should
2486         ** return us the number of supported
2487         ** vectors. (Will be 1 for MSI)
2488         */
2489         adapter->msix = ixgbe_setup_msix(adapter);
2490         return (0);
2491 }
2492
2493 static void
2494 ixgbe_free_pci_resources(struct adapter * adapter)
2495 {
2496         struct          ix_queue *que = adapter->queues;
2497         device_t        dev = adapter->dev;
2498         int             rid, memrid;
2499
2500         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2501                 memrid = PCIR_BAR(MSIX_82598_BAR);
2502         else
2503                 memrid = PCIR_BAR(MSIX_82599_BAR);
2504
2505         /*
2506         ** There is a slight possibility of a failure mode
2507         ** in attach that will result in entering this function
2508         ** before interrupt resources have been initialized, and
2509         ** in that case we do not want to execute the loops below
2510         ** We can detect this reliably by the state of the adapter
2511         ** res pointer.
2512         */
2513         if (adapter->res == NULL)
2514                 goto mem;
2515
2516         /*
2517         **  Release all msix queue resources:
2518         */
2519         for (int i = 0; i < adapter->num_queues; i++, que++) {
2520                 rid = que->msix + 1;
2521                 if (que->tag != NULL) {
2522                         bus_teardown_intr(dev, que->res, que->tag);
2523                         que->tag = NULL;
2524                 }
2525                 if (que->res != NULL)
2526                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2527         }
2528
2529
2530         /* Clean the Legacy or Link interrupt last */
2531         if (adapter->linkvec) /* we are doing MSIX */
2532                 rid = adapter->linkvec + 1;
2533         else
2534                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2535
2536         if (adapter->tag != NULL) {
2537                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2538                 adapter->tag = NULL;
2539         }
2540         if (adapter->res != NULL)
2541                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2542
2543 mem:
2544         if (adapter->msix)
2545                 pci_release_msi(dev);
2546
2547         if (adapter->msix_mem != NULL)
2548                 bus_release_resource(dev, SYS_RES_MEMORY,
2549                     memrid, adapter->msix_mem);
2550
2551         if (adapter->pci_mem != NULL)
2552                 bus_release_resource(dev, SYS_RES_MEMORY,
2553                     PCIR_BAR(0), adapter->pci_mem);
2554
2555         return;
2556 }
2557
2558 /*********************************************************************
2559  *
2560  *  Setup networking device structure and register an interface.
2561  *
2562  **********************************************************************/
2563 static int
2564 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2565 {
2566         struct ixgbe_hw *hw = &adapter->hw;
2567         struct ifnet   *ifp;
2568
2569         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2570
2571         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2572         if (ifp == NULL) {
2573                 device_printf(dev, "can not allocate ifnet structure\n");
2574                 return (-1);
2575         }
2576         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2577         ifp->if_baudrate = 1000000000;
2578         ifp->if_init = ixgbe_init;
2579         ifp->if_softc = adapter;
2580         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2581         ifp->if_ioctl = ixgbe_ioctl;
2582 #ifndef IXGBE_LEGACY_TX
2583         ifp->if_transmit = ixgbe_mq_start;
2584         ifp->if_qflush = ixgbe_qflush;
2585 #else
2586         ifp->if_start = ixgbe_start;
2587         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2588 #endif
2589
2590         ether_ifattach(ifp, adapter->hw.mac.addr);
2591
2592         adapter->max_frame_size =
2593             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2594
2595         /*
2596          * Tell the upper layer(s) we support long frames.
2597          */
2598         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2599
2600         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2601         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2602         ifp->if_capabilities |= IFCAP_LRO;
2603         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2604                              |  IFCAP_VLAN_HWTSO
2605                              |  IFCAP_VLAN_MTU;
2606         ifp->if_capenable = ifp->if_capabilities;
2607
2608         /*
2609         ** Don't turn this on by default, if vlans are
2610         ** created on another pseudo device (eg. lagg)
2611         ** then vlan events are not passed thru, breaking
2612         ** operation, but with HW FILTER off it works. If
2613         ** using vlans directly on the ixgbe driver you can
2614         ** enable this and get full hardware tag filtering.
2615         */
2616         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2617
2618         /*
2619          * Specify the media types supported by this adapter and register
2620          * callbacks to update media and link information
2621          */
2622         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2623                      ixgbe_media_status);
2624         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2625         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2626         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2627                 ifmedia_add(&adapter->media,
2628                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2629                 ifmedia_add(&adapter->media,
2630                     IFM_ETHER | IFM_1000_T, 0, NULL);
2631         }
2632         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2633         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2634
2635         return (0);
2636 }
2637
2638 static void
2639 ixgbe_config_link(struct adapter *adapter)
2640 {
2641         struct ixgbe_hw *hw = &adapter->hw;
2642         u32     autoneg, err = 0;
2643         bool    sfp, negotiate;
2644
2645         sfp = ixgbe_is_sfp(hw);
2646
2647         if (sfp) {
2648                 if (hw->phy.multispeed_fiber) {
2649                         hw->mac.ops.setup_sfp(hw);
2650                         ixgbe_enable_tx_laser(hw);
2651                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2652                 } else
2653                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2654         } else {
2655                 if (hw->mac.ops.check_link)
2656                         err = ixgbe_check_link(hw, &autoneg,
2657                             &adapter->link_up, FALSE);
2658                 if (err)
2659                         goto out;
2660                 autoneg = hw->phy.autoneg_advertised;
2661                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2662                         err  = hw->mac.ops.get_link_capabilities(hw,
2663                             &autoneg, &negotiate);
2664                 if (err)
2665                         goto out;
2666                 if (hw->mac.ops.setup_link)
2667                         err = hw->mac.ops.setup_link(hw, autoneg,
2668                             negotiate, adapter->link_up);
2669         }
2670 out:
2671         return;
2672 }
2673
2674 /********************************************************************
2675  * Manage DMA'able memory.
2676  *******************************************************************/
2677 static void
2678 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2679 {
2680         if (error)
2681                 return;
2682         *(bus_addr_t *) arg = segs->ds_addr;
2683         return;
2684 }
2685
2686 static int
2687 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2688                 struct ixgbe_dma_alloc *dma, int mapflags)
2689 {
2690         device_t dev = adapter->dev;
2691         int             r;
2692
2693         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2694                                DBA_ALIGN, 0,    /* alignment, bounds */
2695                                BUS_SPACE_MAXADDR,       /* lowaddr */
2696                                BUS_SPACE_MAXADDR,       /* highaddr */
2697                                NULL, NULL,      /* filter, filterarg */
2698                                size,    /* maxsize */
2699                                1,       /* nsegments */
2700                                size,    /* maxsegsize */
2701                                BUS_DMA_ALLOCNOW,        /* flags */
2702                                NULL,    /* lockfunc */
2703                                NULL,    /* lockfuncarg */
2704                                &dma->dma_tag);
2705         if (r != 0) {
2706                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2707                        "error %u\n", r);
2708                 goto fail_0;
2709         }
2710         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2711                              BUS_DMA_NOWAIT, &dma->dma_map);
2712         if (r != 0) {
2713                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2714                        "error %u\n", r);
2715                 goto fail_1;
2716         }
2717         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2718                             size,
2719                             ixgbe_dmamap_cb,
2720                             &dma->dma_paddr,
2721                             mapflags | BUS_DMA_NOWAIT);
2722         if (r != 0) {
2723                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2724                        "error %u\n", r);
2725                 goto fail_2;
2726         }
2727         dma->dma_size = size;
2728         return (0);
2729 fail_2:
2730         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2731 fail_1:
2732         bus_dma_tag_destroy(dma->dma_tag);
2733 fail_0:
2734         dma->dma_map = NULL;
2735         dma->dma_tag = NULL;
2736         return (r);
2737 }
2738
2739 static void
2740 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2741 {
2742         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2743             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2744         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2745         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2746         bus_dma_tag_destroy(dma->dma_tag);
2747 }
2748
2749
2750 /*********************************************************************
2751  *
2752  *  Allocate memory for the transmit and receive rings, and then
2753  *  the descriptors associated with each, called only once at attach.
2754  *
2755  **********************************************************************/
2756 static int
2757 ixgbe_allocate_queues(struct adapter *adapter)
2758 {
2759         device_t        dev = adapter->dev;
2760         struct ix_queue *que;
2761         struct tx_ring  *txr;
2762         struct rx_ring  *rxr;
2763         int rsize, tsize, error = IXGBE_SUCCESS;
2764         int txconf = 0, rxconf = 0;
2765
2766         /* First allocate the top level queue structs */
2767         if (!(adapter->queues =
2768             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2769             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2770                 device_printf(dev, "Unable to allocate queue memory\n");
2771                 error = ENOMEM;
2772                 goto fail;
2773         }
2774
2775         /* First allocate the TX ring struct memory */
2776         if (!(adapter->tx_rings =
2777             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2778             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2779                 device_printf(dev, "Unable to allocate TX ring memory\n");
2780                 error = ENOMEM;
2781                 goto tx_fail;
2782         }
2783
2784         /* Next allocate the RX */
2785         if (!(adapter->rx_rings =
2786             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2787             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2788                 device_printf(dev, "Unable to allocate RX ring memory\n");
2789                 error = ENOMEM;
2790                 goto rx_fail;
2791         }
2792
2793         /* For the ring itself */
2794         tsize = roundup2(adapter->num_tx_desc *
2795             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2796
2797         /*
2798          * Now set up the TX queues, txconf is needed to handle the
2799          * possibility that things fail midcourse and we need to
2800          * undo memory gracefully
2801          */
2802         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2803                 /* Set up some basics */
2804                 txr = &adapter->tx_rings[i];
2805                 txr->adapter = adapter;
2806                 txr->me = i;
2807                 txr->num_desc = adapter->num_tx_desc;
2808
2809                 /* Initialize the TX side lock */
2810                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2811                     device_get_nameunit(dev), txr->me);
2812                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2813
2814                 if (ixgbe_dma_malloc(adapter, tsize,
2815                         &txr->txdma, BUS_DMA_NOWAIT)) {
2816                         device_printf(dev,
2817                             "Unable to allocate TX Descriptor memory\n");
2818                         error = ENOMEM;
2819                         goto err_tx_desc;
2820                 }
2821                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2822                 bzero((void *)txr->tx_base, tsize);
2823
2824                 /* Now allocate transmit buffers for the ring */
2825                 if (ixgbe_allocate_transmit_buffers(txr)) {
2826                         device_printf(dev,
2827                             "Critical Failure setting up transmit buffers\n");
2828                         error = ENOMEM;
2829                         goto err_tx_desc;
2830                 }
2831 #ifndef IXGBE_LEGACY_TX
2832                 /* Allocate a buf ring */
2833                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2834                     M_WAITOK, &txr->tx_mtx);
2835                 if (txr->br == NULL) {
2836                         device_printf(dev,
2837                             "Critical Failure setting up buf ring\n");
2838                         error = ENOMEM;
2839                         goto err_tx_desc;
2840                 }
2841 #endif
2842         }
2843
2844         /*
2845          * Next the RX queues...
2846          */
2847         rsize = roundup2(adapter->num_rx_desc *
2848             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2849         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2850                 rxr = &adapter->rx_rings[i];
2851                 /* Set up some basics */
2852                 rxr->adapter = adapter;
2853                 rxr->me = i;
2854                 rxr->num_desc = adapter->num_rx_desc;
2855
2856                 /* Initialize the RX side lock */
2857                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2858                     device_get_nameunit(dev), rxr->me);
2859                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2860
2861                 if (ixgbe_dma_malloc(adapter, rsize,
2862                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2863                         device_printf(dev,
2864                             "Unable to allocate RxDescriptor memory\n");
2865                         error = ENOMEM;
2866                         goto err_rx_desc;
2867                 }
2868                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2869                 bzero((void *)rxr->rx_base, rsize);
2870
2871                 /* Allocate receive buffers for the ring*/
2872                 if (ixgbe_allocate_receive_buffers(rxr)) {
2873                         device_printf(dev,
2874                             "Critical Failure setting up receive buffers\n");
2875                         error = ENOMEM;
2876                         goto err_rx_desc;
2877                 }
2878         }
2879
2880         /*
2881         ** Finally set up the queue holding structs
2882         */
2883         for (int i = 0; i < adapter->num_queues; i++) {
2884                 que = &adapter->queues[i];
2885                 que->adapter = adapter;
2886                 que->txr = &adapter->tx_rings[i];
2887                 que->rxr = &adapter->rx_rings[i];
2888         }
2889
2890         return (0);
2891
2892 err_rx_desc:
2893         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2894                 ixgbe_dma_free(adapter, &rxr->rxdma);
2895 err_tx_desc:
2896         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2897                 ixgbe_dma_free(adapter, &txr->txdma);
2898         free(adapter->rx_rings, M_DEVBUF);
2899 rx_fail:
2900         free(adapter->tx_rings, M_DEVBUF);
2901 tx_fail:
2902         free(adapter->queues, M_DEVBUF);
2903 fail:
2904         return (error);
2905 }
2906
2907 /*********************************************************************
2908  *
2909  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2910  *  the information needed to transmit a packet on the wire. This is
2911  *  called only once at attach, setup is done every reset.
2912  *
2913  **********************************************************************/
2914 static int
2915 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2916 {
2917         struct adapter *adapter = txr->adapter;
2918         device_t dev = adapter->dev;
2919         struct ixgbe_tx_buf *txbuf;
2920         int error, i;
2921
2922         /*
2923          * Setup DMA descriptor areas.
2924          */
2925         if ((error = bus_dma_tag_create(
2926                                bus_get_dma_tag(adapter->dev),   /* parent */
2927                                1, 0,            /* alignment, bounds */
2928                                BUS_SPACE_MAXADDR,       /* lowaddr */
2929                                BUS_SPACE_MAXADDR,       /* highaddr */
2930                                NULL, NULL,              /* filter, filterarg */
2931                                IXGBE_TSO_SIZE,          /* maxsize */
2932                                adapter->num_segs,       /* nsegments */
2933                                PAGE_SIZE,               /* maxsegsize */
2934                                0,                       /* flags */
2935                                NULL,                    /* lockfunc */
2936                                NULL,                    /* lockfuncarg */
2937                                &txr->txtag))) {
2938                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2939                 goto fail;
2940         }
2941
2942         if (!(txr->tx_buffers =
2943             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2944             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2945                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2946                 error = ENOMEM;
2947                 goto fail;
2948         }
2949
2950         /* Create the descriptor buffer dma maps */
2951         txbuf = txr->tx_buffers;
2952         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2953                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2954                 if (error != 0) {
2955                         device_printf(dev, "Unable to create TX DMA map\n");
2956                         goto fail;
2957                 }
2958         }
2959
2960         return 0;
2961 fail:
2962         /* We free all, it handles case where we are in the middle */
2963         ixgbe_free_transmit_structures(adapter);
2964         return (error);
2965 }
2966
2967 /*********************************************************************
2968  *
2969  *  Initialize a transmit ring.
2970  *
2971  **********************************************************************/
2972 static void
2973 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2974 {
2975         struct adapter *adapter = txr->adapter;
2976         struct ixgbe_tx_buf *txbuf;
2977         int i;
2978 #ifdef DEV_NETMAP
2979         struct netmap_adapter *na = NA(adapter->ifp);
2980         struct netmap_slot *slot;
2981 #endif /* DEV_NETMAP */
2982
2983         /* Clear the old ring contents */
2984         IXGBE_TX_LOCK(txr);
2985 #ifdef DEV_NETMAP
2986         /*
2987          * (under lock): if in netmap mode, do some consistency
2988          * checks and set slot to entry 0 of the netmap ring.
2989          */
2990         slot = netmap_reset(na, NR_TX, txr->me, 0);
2991 #endif /* DEV_NETMAP */
2992         bzero((void *)txr->tx_base,
2993               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2994         /* Reset indices */
2995         txr->next_avail_desc = 0;
2996         txr->next_to_clean = 0;
2997
2998         /* Free any existing tx buffers. */
2999         txbuf = txr->tx_buffers;
3000         for (i = 0; i < txr->num_desc; i++, txbuf++) {
3001                 if (txbuf->m_head != NULL) {
3002                         bus_dmamap_sync(txr->txtag, txbuf->map,
3003                             BUS_DMASYNC_POSTWRITE);
3004                         bus_dmamap_unload(txr->txtag, txbuf->map);
3005                         m_freem(txbuf->m_head);
3006                         txbuf->m_head = NULL;
3007                 }
3008 #ifdef DEV_NETMAP
3009                 /*
3010                  * In netmap mode, set the map for the packet buffer.
3011                  * NOTE: Some drivers (not this one) also need to set
3012                  * the physical buffer address in the NIC ring.
3013                  * Slots in the netmap ring (indexed by "si") are
3014                  * kring->nkr_hwofs positions "ahead" wrt the
3015                  * corresponding slot in the NIC ring. In some drivers
3016                  * (not here) nkr_hwofs can be negative. Function
3017                  * netmap_idx_n2k() handles wraparounds properly.
3018                  */
3019                 if (slot) {
3020                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3021                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3022                 }
3023 #endif /* DEV_NETMAP */
3024                 /* Clear the EOP descriptor pointer */
3025                 txbuf->eop = NULL;
3026         }
3027
3028 #ifdef IXGBE_FDIR
3029         /* Set the rate at which we sample packets */
3030         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3031                 txr->atr_sample = atr_sample_rate;
3032 #endif
3033
3034         /* Set number of descriptors available */
3035         txr->tx_avail = adapter->num_tx_desc;
3036
3037         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3038             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3039         IXGBE_TX_UNLOCK(txr);
3040 }
3041
3042 /*********************************************************************
3043  *
3044  *  Initialize all transmit rings.
3045  *
3046  **********************************************************************/
3047 static int
3048 ixgbe_setup_transmit_structures(struct adapter *adapter)
3049 {
3050         struct tx_ring *txr = adapter->tx_rings;
3051
3052         for (int i = 0; i < adapter->num_queues; i++, txr++)
3053                 ixgbe_setup_transmit_ring(txr);
3054
3055         return (0);
3056 }
3057
3058 /*********************************************************************
3059  *
3060  *  Enable transmit unit.
3061  *
3062  **********************************************************************/
3063 static void
3064 ixgbe_initialize_transmit_units(struct adapter *adapter)
3065 {
3066         struct tx_ring  *txr = adapter->tx_rings;
3067         struct ixgbe_hw *hw = &adapter->hw;
3068
3069         /* Setup the Base and Length of the Tx Descriptor Ring */
3070
3071         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3072                 u64     tdba = txr->txdma.dma_paddr;
3073                 u32     txctrl;
3074
3075                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3076                        (tdba & 0x00000000ffffffffULL));
3077                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3078                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3079                     adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3080
3081                 /* Setup the HW Tx Head and Tail descriptor pointers */
3082                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3083                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3084
3085                 /* Setup Transmit Descriptor Cmd Settings */
3086                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3087                 txr->queue_status = IXGBE_QUEUE_IDLE;
3088
3089                 /* Set the processing limit */
3090                 txr->process_limit = ixgbe_tx_process_limit;
3091
3092                 /* Disable Head Writeback */
3093                 switch (hw->mac.type) {
3094                 case ixgbe_mac_82598EB:
3095                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3096                         break;
3097                 case ixgbe_mac_82599EB:
3098                 case ixgbe_mac_X540:
3099                 default:
3100                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3101                         break;
3102                 }
3103                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3104                 switch (hw->mac.type) {
3105                 case ixgbe_mac_82598EB:
3106                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3107                         break;
3108                 case ixgbe_mac_82599EB:
3109                 case ixgbe_mac_X540:
3110                 default:
3111                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3112                         break;
3113                 }
3114
3115         }
3116
3117         if (hw->mac.type != ixgbe_mac_82598EB) {
3118                 u32 dmatxctl, rttdcs;
3119                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3120                 dmatxctl |= IXGBE_DMATXCTL_TE;
3121                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3122                 /* Disable arbiter to set MTQC */
3123                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3124                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3125                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3126                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3127                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3128                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3129         }
3130
3131         return;
3132 }
3133
3134 /*********************************************************************
3135  *
3136  *  Free all transmit rings.
3137  *
3138  **********************************************************************/
3139 static void
3140 ixgbe_free_transmit_structures(struct adapter *adapter)
3141 {
3142         struct tx_ring *txr = adapter->tx_rings;
3143
3144         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3145                 IXGBE_TX_LOCK(txr);
3146                 ixgbe_free_transmit_buffers(txr);
3147                 ixgbe_dma_free(adapter, &txr->txdma);
3148                 IXGBE_TX_UNLOCK(txr);
3149                 IXGBE_TX_LOCK_DESTROY(txr);
3150         }
3151         free(adapter->tx_rings, M_DEVBUF);
3152 }
3153
3154 /*********************************************************************
3155  *
3156  *  Free transmit ring related data structures.
3157  *
3158  **********************************************************************/
3159 static void
3160 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3161 {
3162         struct adapter *adapter = txr->adapter;
3163         struct ixgbe_tx_buf *tx_buffer;
3164         int             i;
3165
3166         INIT_DEBUGOUT("free_transmit_ring: begin");
3167
3168         if (txr->tx_buffers == NULL)
3169                 return;
3170
3171         tx_buffer = txr->tx_buffers;
3172         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3173                 if (tx_buffer->m_head != NULL) {
3174                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3175                             BUS_DMASYNC_POSTWRITE);
3176                         bus_dmamap_unload(txr->txtag,
3177                             tx_buffer->map);
3178                         m_freem(tx_buffer->m_head);
3179                         tx_buffer->m_head = NULL;
3180                         if (tx_buffer->map != NULL) {
3181                                 bus_dmamap_destroy(txr->txtag,
3182                                     tx_buffer->map);
3183                                 tx_buffer->map = NULL;
3184                         }
3185                 } else if (tx_buffer->map != NULL) {
3186                         bus_dmamap_unload(txr->txtag,
3187                             tx_buffer->map);
3188                         bus_dmamap_destroy(txr->txtag,
3189                             tx_buffer->map);
3190                         tx_buffer->map = NULL;
3191                 }
3192         }
3193 #ifdef IXGBE_LEGACY_TX
3194         if (txr->br != NULL)
3195                 buf_ring_free(txr->br, M_DEVBUF);
3196 #endif
3197         if (txr->tx_buffers != NULL) {
3198                 free(txr->tx_buffers, M_DEVBUF);
3199                 txr->tx_buffers = NULL;
3200         }
3201         if (txr->txtag != NULL) {
3202                 bus_dma_tag_destroy(txr->txtag);
3203                 txr->txtag = NULL;
3204         }
3205         return;
3206 }
3207
3208 /*********************************************************************
3209  *
3210  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
3211  *
3212  **********************************************************************/
3213
3214 static int
3215 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3216     u32 *cmd_type_len, u32 *olinfo_status)
3217 {
3218         struct ixgbe_adv_tx_context_desc *TXD;
3219         struct ether_vlan_header *eh;
3220         struct ip *ip;
3221         struct ip6_hdr *ip6;
3222         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3223         int     ehdrlen, ip_hlen = 0;
3224         u16     etype;
3225         u8      ipproto = 0;
3226         int     offload = TRUE;
3227         int     ctxd = txr->next_avail_desc;
3228         u16     vtag = 0;
3229
3230         /* First check if TSO is to be used */
3231         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3232                 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3233
3234         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3235                 offload = FALSE;
3236
3237         /* Indicate the whole packet as payload when not doing TSO */
3238         *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3239
3240         /* Now ready a context descriptor */
3241         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3242
3243         /*
3244         ** In advanced descriptors the vlan tag must
3245         ** be placed into the context descriptor. Hence
3246         ** we need to make one even if not doing offloads.
3247         */
3248         if (mp->m_flags & M_VLANTAG) {
3249                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3250                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3251         } else if (offload == FALSE) /* ... no offload to do */
3252                 return (0);
3253
3254         /*
3255          * Determine where frame payload starts.
3256          * Jump over vlan headers if already present,
3257          * helpful for QinQ too.
3258          */
3259         eh = mtod(mp, struct ether_vlan_header *);
3260         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3261                 etype = ntohs(eh->evl_proto);
3262                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3263         } else {
3264                 etype = ntohs(eh->evl_encap_proto);
3265                 ehdrlen = ETHER_HDR_LEN;
3266         }
3267
3268         /* Set the ether header length */
3269         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3270
3271         switch (etype) {
3272                 case ETHERTYPE_IP:
3273                         ip = (struct ip *)(mp->m_data + ehdrlen);
3274                         ip_hlen = ip->ip_hl << 2;
3275                         ipproto = ip->ip_p;
3276                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3277                         break;
3278                 case ETHERTYPE_IPV6:
3279                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3280                         ip_hlen = sizeof(struct ip6_hdr);
3281                         ipproto = ip6->ip6_nxt;
3282                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3283                         break;
3284                 default:
3285                         offload = FALSE;
3286                         break;
3287         }
3288
3289         vlan_macip_lens |= ip_hlen;
3290         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3291
3292         switch (ipproto) {
3293                 case IPPROTO_TCP:
3294                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3295                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3296                         break;
3297
3298                 case IPPROTO_UDP:
3299                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3300                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3301                         break;
3302
3303 #if __FreeBSD_version >= 800000
3304                 case IPPROTO_SCTP:
3305                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3306                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3307                         break;
3308 #endif
3309                 default:
3310                         offload = FALSE;
3311                         break;
3312         }
3313
3314         if (offload) /* For the TX descriptor setup */
3315                 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3316
3317         /* Now copy bits into descriptor */
3318         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3319         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3320         TXD->seqnum_seed = htole32(0);
3321         TXD->mss_l4len_idx = htole32(0);
3322
3323         /* We've consumed the first desc, adjust counters */
3324         if (++ctxd == txr->num_desc)
3325                 ctxd = 0;
3326         txr->next_avail_desc = ctxd;
3327         --txr->tx_avail;
3328
3329         return (0);
3330 }
3331
3332 /**********************************************************************
3333  *
3334  *  Setup work for hardware segmentation offload (TSO) on
3335  *  adapters using advanced tx descriptors
3336  *
3337  **********************************************************************/
3338 static int
3339 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3340     u32 *cmd_type_len, u32 *olinfo_status)
3341 {
3342         struct ixgbe_adv_tx_context_desc *TXD;
3343         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3344         u32 mss_l4len_idx = 0, paylen;
3345         u16 vtag = 0;
3346         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3347         struct ether_vlan_header *eh;
3348         struct ip *ip;
3349         struct tcphdr *th;
3350
3351
3352         /*
3353          * Determine where frame payload starts.
3354          * Jump over vlan headers if already present
3355          */
3356         eh = mtod(mp, struct ether_vlan_header *);
3357         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3358                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3359         else
3360                 ehdrlen = ETHER_HDR_LEN;
3361
3362         ip = (struct ip *)(mp->m_data + ehdrlen);
3363         if (ip->ip_p != IPPROTO_TCP)
3364                 return (ENXIO);
3365         ip->ip_sum = 0;
3366         ip_hlen = ip->ip_hl << 2;
3367         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3368         th->th_sum = in_pseudo(ip->ip_src.s_addr,
3369             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3370         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3371         /* Tell transmit desc to also do IPv4 checksum. */
3372         *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3373
3374         ctxd = txr->next_avail_desc;
3375         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3376
3377         tcp_hlen = th->th_off << 2;
3378
3379         /* This is used in the transmit desc in encap */
3380         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3381
3382         /* VLAN MACLEN IPLEN */
3383         if (mp->m_flags & M_VLANTAG) {
3384                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3385                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3386         }
3387
3388         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3389         vlan_macip_lens |= ip_hlen;
3390         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3391
3392         /* ADV DTYPE TUCMD */
3393         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3394         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3395         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3396
3397         /* MSS L4LEN IDX */
3398         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3399         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3400         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3401
3402         TXD->seqnum_seed = htole32(0);
3403
3404         if (++ctxd == txr->num_desc)
3405                 ctxd = 0;
3406
3407         txr->tx_avail--;
3408         txr->next_avail_desc = ctxd;
3409         *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3410         *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3411         *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3412         ++txr->tso_tx;
3413         return (0);
3414 }
3415
3416 #ifdef IXGBE_FDIR
3417 /*
3418 ** This routine parses packet headers so that Flow
3419 ** Director can make a hashed filter table entry
3420 ** allowing traffic flows to be identified and kept
3421 ** on the same cpu.  This would be a performance
3422 ** hit, but we only do it at IXGBE_FDIR_RATE of
3423 ** packets.
3424 */
3425 static void
3426 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3427 {
3428         struct adapter                  *adapter = txr->adapter;
3429         struct ix_queue                 *que;
3430         struct ip                       *ip;
3431         struct tcphdr                   *th;
3432         struct udphdr                   *uh;
3433         struct ether_vlan_header        *eh;
3434         union ixgbe_atr_hash_dword      input = {.dword = 0};
3435         union ixgbe_atr_hash_dword      common = {.dword = 0};
3436         int                             ehdrlen, ip_hlen;
3437         u16                             etype;
3438
3439         eh = mtod(mp, struct ether_vlan_header *);
3440         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3441                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3442                 etype = eh->evl_proto;
3443         } else {
3444                 ehdrlen = ETHER_HDR_LEN;
3445                 etype = eh->evl_encap_proto;
3446         }
3447
3448         /* Only handling IPv4 */
3449         if (etype != htons(ETHERTYPE_IP))
3450                 return;
3451
3452         ip = (struct ip *)(mp->m_data + ehdrlen);
3453         ip_hlen = ip->ip_hl << 2;
3454
3455         /* check if we're UDP or TCP */
3456         switch (ip->ip_p) {
3457         case IPPROTO_TCP:
3458                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3459                 /* src and dst are inverted */
3460                 common.port.dst ^= th->th_sport;
3461                 common.port.src ^= th->th_dport;
3462                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3463                 break;
3464         case IPPROTO_UDP:
3465                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3466                 /* src and dst are inverted */
3467                 common.port.dst ^= uh->uh_sport;
3468                 common.port.src ^= uh->uh_dport;
3469                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3470                 break;
3471         default:
3472                 return;
3473         }
3474
3475         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3476         if (mp->m_pkthdr.ether_vtag)
3477                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3478         else
3479                 common.flex_bytes ^= etype;
3480         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3481
3482         que = &adapter->queues[txr->me];
3483         /*
3484         ** This assumes the Rx queue and Tx
3485         ** queue are bound to the same CPU
3486         */
3487         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3488             input, common, que->msix);
3489 }
3490 #endif /* IXGBE_FDIR */
3491
3492 /**********************************************************************
3493  *
3494  *  Examine each tx_buffer in the used queue. If the hardware is done
3495  *  processing the packet then free associated resources. The
3496  *  tx_buffer is put back on the free queue.
3497  *
3498  **********************************************************************/
3499 static bool
3500 ixgbe_txeof(struct tx_ring *txr)
3501 {
3502         struct adapter          *adapter = txr->adapter;
3503         struct ifnet            *ifp = adapter->ifp;
3504         u32                     work, processed = 0;
3505         u16                     limit = txr->process_limit;
3506         struct ixgbe_tx_buf     *buf;
3507         union ixgbe_adv_tx_desc *txd;
3508
3509         mtx_assert(&txr->tx_mtx, MA_OWNED);
3510
3511 #ifdef DEV_NETMAP
3512         if (ifp->if_capenable & IFCAP_NETMAP) {
3513                 struct netmap_adapter *na = NA(ifp);
3514                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3515                 txd = txr->tx_base;
3516                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3517                     BUS_DMASYNC_POSTREAD);
3518                 /*
3519                  * In netmap mode, all the work is done in the context
3520                  * of the client thread. Interrupt handlers only wake up
3521                  * clients, which may be sleeping on individual rings
3522                  * or on a global resource for all rings.
3523                  * To implement tx interrupt mitigation, we wake up the client
3524                  * thread roughly every half ring, even if the NIC interrupts
3525                  * more frequently. This is implemented as follows:
3526                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3527                  *   the slot that should wake up the thread (nkr_num_slots
3528                  *   means the user thread should not be woken up);
3529                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3530                  *   or the slot has the DD bit set.
3531                  *
3532                  * When the driver has separate locks, we need to
3533                  * release and re-acquire txlock to avoid deadlocks.
3534                  * XXX see if we can find a better way.
3535                  */
3536                 if (!netmap_mitigate ||
3537                     (kring->nr_kflags < kring->nkr_num_slots &&
3538                     txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3539                         kring->nr_kflags = kring->nkr_num_slots;
3540                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3541                         IXGBE_TX_UNLOCK(txr);
3542                         IXGBE_CORE_LOCK(adapter);
3543                         selwakeuppri(&na->tx_si, PI_NET);
3544                         IXGBE_CORE_UNLOCK(adapter);
3545                         IXGBE_TX_LOCK(txr);
3546                 }
3547                 return FALSE;
3548         }
3549 #endif /* DEV_NETMAP */
3550
3551         if (txr->tx_avail == txr->num_desc) {
3552                 txr->queue_status = IXGBE_QUEUE_IDLE;
3553                 return FALSE;
3554         }
3555
3556         /* Get work starting point */
3557         work = txr->next_to_clean;
3558         buf = &txr->tx_buffers[work];
3559         txd = &txr->tx_base[work];
3560         work -= txr->num_desc; /* The distance to ring end */
3561         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3562             BUS_DMASYNC_POSTREAD);
3563
3564         do {
3565                 union ixgbe_adv_tx_desc *eop= buf->eop;
3566                 if (eop == NULL) /* No work */
3567                         break;
3568
3569                 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3570                         break;  /* I/O not complete */
3571
3572                 if (buf->m_head) {
3573                         txr->bytes +=
3574                             buf->m_head->m_pkthdr.len;
3575                         bus_dmamap_sync(txr->txtag,
3576                             buf->map,
3577                             BUS_DMASYNC_POSTWRITE);
3578                         bus_dmamap_unload(txr->txtag,
3579                             buf->map);
3580                         m_freem(buf->m_head);
3581                         buf->m_head = NULL;
3582                         buf->map = NULL;
3583                 }
3584                 buf->eop = NULL;
3585                 ++txr->tx_avail;
3586
3587                 /* We clean the range if multi segment */
3588                 while (txd != eop) {
3589                         ++txd;
3590                         ++buf;
3591                         ++work;
3592                         /* wrap the ring? */
3593                         if (__predict_false(!work)) {
3594                                 work -= txr->num_desc;
3595                                 buf = txr->tx_buffers;
3596                                 txd = txr->tx_base;
3597                         }
3598                         if (buf->m_head) {
3599                                 txr->bytes +=
3600                                     buf->m_head->m_pkthdr.len;
3601                                 bus_dmamap_sync(txr->txtag,
3602                                     buf->map,
3603                                     BUS_DMASYNC_POSTWRITE);
3604                                 bus_dmamap_unload(txr->txtag,
3605                                     buf->map);
3606                                 m_freem(buf->m_head);
3607                                 buf->m_head = NULL;
3608                                 buf->map = NULL;
3609                         }
3610                         ++txr->tx_avail;
3611                         buf->eop = NULL;
3612
3613                 }
3614                 ++txr->packets;
3615                 ++processed;
3616                 ++ifp->if_opackets;
3617                 txr->watchdog_time = ticks;
3618
3619                 /* Try the next packet */
3620                 ++txd;
3621                 ++buf;
3622                 ++work;
3623                 /* reset with a wrap */
3624                 if (__predict_false(!work)) {
3625                         work -= txr->num_desc;
3626                         buf = txr->tx_buffers;
3627                         txd = txr->tx_base;
3628                 }
3629                 prefetch(txd);
3630         } while (__predict_true(--limit));
3631
3632         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3633             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3634
3635         work += txr->num_desc;
3636         txr->next_to_clean = work;
3637
3638         /*
3639         ** Watchdog calculation, we know there's
3640         ** work outstanding or the first return
3641         ** would have been taken, so none processed
3642         ** for too long indicates a hang.
3643         */
3644         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3645                 txr->queue_status = IXGBE_QUEUE_HUNG;
3646
3647         if (txr->tx_avail == txr->num_desc) {
3648                 txr->queue_status = IXGBE_QUEUE_IDLE;
3649                 return (FALSE);
3650         }
3651
3652         return TRUE;
3653 }
3654
3655 /*********************************************************************
3656  *
3657  *  Refresh mbuf buffers for RX descriptor rings
3658  *   - now keeps its own state so discards due to resource
3659  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3660  *     it just returns, keeping its placeholder, thus it can simply
3661  *     be recalled to try again.
3662  *
3663  **********************************************************************/
3664 static void
3665 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3666 {
3667         struct adapter          *adapter = rxr->adapter;
3668         bus_dma_segment_t       seg[1];
3669         struct ixgbe_rx_buf     *rxbuf;
3670         struct mbuf             *mp;
3671         int                     i, j, nsegs, error;
3672         bool                    refreshed = FALSE;
3673
3674         i = j = rxr->next_to_refresh;
3675         /* Control the loop with one beyond */
3676         if (++j == rxr->num_desc)
3677                 j = 0;
3678
3679         while (j != limit) {
3680                 rxbuf = &rxr->rx_buffers[i];
3681                 if (rxbuf->buf == NULL) {
3682                         mp = m_getjcl(M_NOWAIT, MT_DATA,
3683                             M_PKTHDR, rxr->mbuf_sz);
3684                         if (mp == NULL)
3685                                 goto update;
3686                 } else
3687                         mp = rxbuf->buf;
3688
3689                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3690
3691                 /* If we're dealing with an mbuf that was copied rather
3692                  * than replaced, there's no need to go through busdma.
3693                  */
3694                 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3695                         /* Get the memory mapping */
3696                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3697                             rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3698                         if (error != 0) {
3699                                 printf("Refresh mbufs: payload dmamap load"
3700                                     " failure - %d\n", error);
3701                                 m_free(mp);
3702                                 rxbuf->buf = NULL;
3703                                 goto update;
3704                         }
3705                         rxbuf->buf = mp;
3706                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3707                             BUS_DMASYNC_PREREAD);
3708                         rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3709                             htole64(seg[0].ds_addr);
3710                 } else {
3711                         rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3712                         rxbuf->flags &= ~IXGBE_RX_COPY;
3713                 }
3714
3715                 refreshed = TRUE;
3716                 /* Next is precalculated */
3717                 i = j;
3718                 rxr->next_to_refresh = i;
3719                 if (++j == rxr->num_desc)
3720                         j = 0;
3721         }
3722 update:
3723         if (refreshed) /* Update hardware tail index */
3724                 IXGBE_WRITE_REG(&adapter->hw,
3725                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3726         return;
3727 }
3728
3729 /*********************************************************************
3730  *
3731  *  Allocate memory for rx_buffer structures. Since we use one
3732  *  rx_buffer per received packet, the maximum number of rx_buffer's
3733  *  that we'll need is equal to the number of receive descriptors
3734  *  that we've allocated.
3735  *
3736  **********************************************************************/
3737 static int
3738 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3739 {
3740         struct  adapter         *adapter = rxr->adapter;
3741         device_t                dev = adapter->dev;
3742         struct ixgbe_rx_buf     *rxbuf;
3743         int                     i, bsize, error;
3744
3745         bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3746         if (!(rxr->rx_buffers =
3747             (struct ixgbe_rx_buf *) malloc(bsize,
3748             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3749                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3750                 error = ENOMEM;
3751                 goto fail;
3752         }
3753
3754         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3755                                    1, 0,        /* alignment, bounds */
3756                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3757                                    BUS_SPACE_MAXADDR,   /* highaddr */
3758                                    NULL, NULL,          /* filter, filterarg */
3759                                    MJUM16BYTES,         /* maxsize */
3760                                    1,                   /* nsegments */
3761                                    MJUM16BYTES,         /* maxsegsize */
3762                                    0,                   /* flags */
3763                                    NULL,                /* lockfunc */
3764                                    NULL,                /* lockfuncarg */
3765                                    &rxr->ptag))) {
3766                 device_printf(dev, "Unable to create RX DMA tag\n");
3767                 goto fail;
3768         }
3769
3770         for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3771                 rxbuf = &rxr->rx_buffers[i];
3772                 error = bus_dmamap_create(rxr->ptag,
3773                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3774                 if (error) {
3775                         device_printf(dev, "Unable to create RX dma map\n");
3776                         goto fail;
3777                 }
3778         }
3779
3780         return (0);
3781
3782 fail:
3783         /* Frees all, but can handle partial completion */
3784         ixgbe_free_receive_structures(adapter);
3785         return (error);
3786 }
3787
3788 /*
3789 ** Used to detect a descriptor that has
3790 ** been merged by Hardware RSC.
3791 */
3792 static inline u32
3793 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3794 {
3795         return (le32toh(rx->wb.lower.lo_dword.data) &
3796             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3797 }
3798
3799 /*********************************************************************
3800  *
3801  *  Initialize Hardware RSC (LRO) feature on 82599
3802  *  for an RX ring, this is toggled by the LRO capability
3803  *  even though it is transparent to the stack.
3804  *
3805  *  NOTE: since this HW feature only works with IPV4 and
3806  *        our testing has shown soft LRO to be as effective
3807  *        I have decided to disable this by default.
3808  *
3809  **********************************************************************/
3810 static void
3811 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3812 {
3813         struct  adapter         *adapter = rxr->adapter;
3814         struct  ixgbe_hw        *hw = &adapter->hw;
3815         u32                     rscctrl, rdrxctl;
3816
3817         /* If turning LRO/RSC off we need to disable it */
3818         if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3819                 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3820                 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3821                 return;
3822         }
3823
3824         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3825         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3826 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3827         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3828 #endif /* DEV_NETMAP */
3829         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3830         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3831         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3832
3833         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3834         rscctrl |= IXGBE_RSCCTL_RSCEN;
3835         /*
3836         ** Limit the total number of descriptors that
3837         ** can be combined, so it does not exceed 64K
3838         */
3839         if (rxr->mbuf_sz == MCLBYTES)
3840                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3841         else if (rxr->mbuf_sz == MJUMPAGESIZE)
3842                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3843         else if (rxr->mbuf_sz == MJUM9BYTES)
3844                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3845         else  /* Using 16K cluster */
3846                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3847
3848         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3849
3850         /* Enable TCP header recognition */
3851         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3852             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3853             IXGBE_PSRTYPE_TCPHDR));
3854
3855         /* Disable RSC for ACK packets */
3856         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3857             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3858
3859         rxr->hw_rsc = TRUE;
3860 }
3861
3862
3863 static void
3864 ixgbe_free_receive_ring(struct rx_ring *rxr)
3865 {
3866         struct ixgbe_rx_buf       *rxbuf;
3867         int i;
3868
3869         for (i = 0; i < rxr->num_desc; i++) {
3870                 rxbuf = &rxr->rx_buffers[i];
3871                 if (rxbuf->buf != NULL) {
3872                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3873                             BUS_DMASYNC_POSTREAD);
3874                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3875                         rxbuf->buf->m_flags |= M_PKTHDR;
3876                         m_freem(rxbuf->buf);
3877                         rxbuf->buf = NULL;
3878                 }
3879         }
3880 }
3881
3882
3883 /*********************************************************************
3884  *
3885  *  Initialize a receive ring and its buffers.
3886  *
3887  **********************************************************************/
3888 static int
3889 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3890 {
3891         struct  adapter         *adapter;
3892         struct ifnet            *ifp;
3893         device_t                dev;
3894         struct ixgbe_rx_buf     *rxbuf;
3895         bus_dma_segment_t       seg[1];
3896         struct lro_ctrl         *lro = &rxr->lro;
3897         int                     rsize, nsegs, error = 0;
3898 #ifdef DEV_NETMAP
3899         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3900         struct netmap_slot *slot;
3901 #endif /* DEV_NETMAP */
3902
3903         adapter = rxr->adapter;
3904         ifp = adapter->ifp;
3905         dev = adapter->dev;
3906
3907         /* Clear the ring contents */
3908         IXGBE_RX_LOCK(rxr);
3909 #ifdef DEV_NETMAP
3910         /* same as in ixgbe_setup_transmit_ring() */
3911         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3912 #endif /* DEV_NETMAP */
3913         rsize = roundup2(adapter->num_rx_desc *
3914             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3915         bzero((void *)rxr->rx_base, rsize);
3916         /* Cache the size */
3917         rxr->mbuf_sz = adapter->rx_mbuf_sz;
3918
3919         /* Free current RX buffer structs and their mbufs */
3920         ixgbe_free_receive_ring(rxr);
3921
3922         /* Now replenish the mbufs */
3923         for (int j = 0; j != rxr->num_desc; ++j) {
3924                 struct mbuf     *mp;
3925
3926                 rxbuf = &rxr->rx_buffers[j];
3927 #ifdef DEV_NETMAP
3928                 /*
3929                  * In netmap mode, fill the map and set the buffer
3930                  * address in the NIC ring, considering the offset
3931                  * between the netmap and NIC rings (see comment in
3932                  * ixgbe_setup_transmit_ring() ). No need to allocate
3933                  * an mbuf, so end the block with a continue;
3934                  */
3935                 if (slot) {
3936                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3937                         uint64_t paddr;
3938                         void *addr;
3939
3940                         addr = PNMB(slot + sj, &paddr);
3941                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3942                         /* Update descriptor */
3943                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3944                         continue;
3945                 }
3946 #endif /* DEV_NETMAP */
3947                 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
3948                     M_PKTHDR, adapter->rx_mbuf_sz);
3949                 if (rxbuf->buf == NULL) {
3950                         error = ENOBUFS;
3951                         goto fail;
3952                 }
3953                 mp = rxbuf->buf;
3954                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3955                 /* Get the memory mapping */
3956                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3957                     rxbuf->pmap, mp, seg,
3958                     &nsegs, BUS_DMA_NOWAIT);
3959                 if (error != 0)
3960                         goto fail;
3961                 bus_dmamap_sync(rxr->ptag,
3962                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3963                 /* Update descriptor */
3964                 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
3965         }
3966
3967
3968         /* Setup our descriptor indices */
3969         rxr->next_to_check = 0;
3970         rxr->next_to_refresh = 0;
3971         rxr->lro_enabled = FALSE;
3972         rxr->rx_copies = 0;
3973         rxr->rx_bytes = 0;
3974         rxr->discard = FALSE;
3975         rxr->vtag_strip = FALSE;
3976
3977         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3978             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3979
3980         /*
3981         ** Now set up the LRO interface:
3982         */
3983         if (ixgbe_rsc_enable)
3984                 ixgbe_setup_hw_rsc(rxr);
3985         else if (ifp->if_capenable & IFCAP_LRO) {
3986                 int err = tcp_lro_init(lro);
3987                 if (err) {
3988                         device_printf(dev, "LRO Initialization failed!\n");
3989                         goto fail;
3990                 }
3991                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3992                 rxr->lro_enabled = TRUE;
3993                 lro->ifp = adapter->ifp;
3994         }
3995
3996         IXGBE_RX_UNLOCK(rxr);
3997         return (0);
3998
3999 fail:
4000         ixgbe_free_receive_ring(rxr);
4001         IXGBE_RX_UNLOCK(rxr);
4002         return (error);
4003 }
4004
4005 /*********************************************************************
4006  *
4007  *  Initialize all receive rings.
4008  *
4009  **********************************************************************/
4010 static int
4011 ixgbe_setup_receive_structures(struct adapter *adapter)
4012 {
4013         struct rx_ring *rxr = adapter->rx_rings;
4014         int j;
4015
4016         for (j = 0; j < adapter->num_queues; j++, rxr++)
4017                 if (ixgbe_setup_receive_ring(rxr))
4018                         goto fail;
4019
4020         return (0);
4021 fail:
4022         /*
4023          * Free RX buffers allocated so far, we will only handle
4024          * the rings that completed, the failing case will have
4025          * cleaned up for itself. 'j' failed, so its the terminus.
4026          */
4027         for (int i = 0; i < j; ++i) {
4028                 rxr = &adapter->rx_rings[i];
4029                 ixgbe_free_receive_ring(rxr);
4030         }
4031
4032         return (ENOBUFS);
4033 }
4034
4035 /*********************************************************************
4036  *
4037  *  Setup receive registers and features.
4038  *
4039  **********************************************************************/
4040 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4041
4042 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4043
4044 static void
4045 ixgbe_initialize_receive_units(struct adapter *adapter)
4046 {
4047         struct  rx_ring *rxr = adapter->rx_rings;
4048         struct ixgbe_hw *hw = &adapter->hw;
4049         struct ifnet   *ifp = adapter->ifp;
4050         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4051         u32             reta, mrqc = 0, hlreg, random[10];
4052
4053
4054         /*
4055          * Make sure receives are disabled while
4056          * setting up the descriptor ring
4057          */
4058         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4059         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4060             rxctrl & ~IXGBE_RXCTRL_RXEN);
4061
4062         /* Enable broadcasts */
4063         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4064         fctrl |= IXGBE_FCTRL_BAM;
4065         fctrl |= IXGBE_FCTRL_DPF;
4066         fctrl |= IXGBE_FCTRL_PMCF;
4067         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4068
4069         /* Set for Jumbo Frames? */
4070         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4071         if (ifp->if_mtu > ETHERMTU)
4072                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4073         else
4074                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4075 #ifdef DEV_NETMAP
4076         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4077         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4078                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4079         else
4080                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4081 #endif /* DEV_NETMAP */
4082         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4083
4084         bufsz = (adapter->rx_mbuf_sz +
4085             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4086
4087         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4088                 u64 rdba = rxr->rxdma.dma_paddr;
4089
4090                 /* Setup the Base and Length of the Rx Descriptor Ring */
4091                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4092                                (rdba & 0x00000000ffffffffULL));
4093                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4094                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4095                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4096
4097                 /* Set up the SRRCTL register */
4098                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4099                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4100                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4101                 srrctl |= bufsz;
4102                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4103                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4104
4105                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4106                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4107                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4108
4109                 /* Set the processing limit */
4110                 rxr->process_limit = ixgbe_rx_process_limit;
4111         }
4112
4113         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4114                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4115                               IXGBE_PSRTYPE_UDPHDR |
4116                               IXGBE_PSRTYPE_IPV4HDR |
4117                               IXGBE_PSRTYPE_IPV6HDR;
4118                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4119         }
4120
4121         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4122
4123         /* Setup RSS */
4124         if (adapter->num_queues > 1) {
4125                 int i, j;
4126                 reta = 0;
4127
4128                 /* set up random bits */
4129                 arc4rand(&random, sizeof(random), 0);
4130
4131                 /* Set up the redirection table */
4132                 for (i = 0, j = 0; i < 128; i++, j++) {
4133                         if (j == adapter->num_queues) j = 0;
4134                         reta = (reta << 8) | (j * 0x11);
4135                         if ((i & 3) == 3)
4136                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4137                 }
4138
4139                 /* Now fill our hash function seeds */
4140                 for (int i = 0; i < 10; i++)
4141                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4142
4143                 /* Perform hash on these packet types */
4144                 mrqc = IXGBE_MRQC_RSSEN
4145                      | IXGBE_MRQC_RSS_FIELD_IPV4
4146                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4147                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4148                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4149                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4150                      | IXGBE_MRQC_RSS_FIELD_IPV6
4151                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4152                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4153                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4154                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4155
4156                 /* RSS and RX IPP Checksum are mutually exclusive */
4157                 rxcsum |= IXGBE_RXCSUM_PCSD;
4158         }
4159
4160         if (ifp->if_capenable & IFCAP_RXCSUM)
4161                 rxcsum |= IXGBE_RXCSUM_PCSD;
4162
4163         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4164                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4165
4166         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4167
4168         return;
4169 }
4170
4171 /*********************************************************************
4172  *
4173  *  Free all receive rings.
4174  *
4175  **********************************************************************/
4176 static void
4177 ixgbe_free_receive_structures(struct adapter *adapter)
4178 {
4179         struct rx_ring *rxr = adapter->rx_rings;
4180
4181         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4182                 struct lro_ctrl         *lro = &rxr->lro;
4183                 ixgbe_free_receive_buffers(rxr);
4184                 /* Free LRO memory */
4185                 tcp_lro_free(lro);
4186                 /* Free the ring memory as well */
4187                 ixgbe_dma_free(adapter, &rxr->rxdma);
4188         }
4189
4190         free(adapter->rx_rings, M_DEVBUF);
4191 }
4192
4193
4194 /*********************************************************************
4195  *
4196  *  Free receive ring data structures
4197  *
4198  **********************************************************************/
4199 static void
4200 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4201 {
4202         struct adapter          *adapter = rxr->adapter;
4203         struct ixgbe_rx_buf     *rxbuf;
4204
4205         INIT_DEBUGOUT("free_receive_structures: begin");
4206
4207         /* Cleanup any existing buffers */
4208         if (rxr->rx_buffers != NULL) {
4209                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4210                         rxbuf = &rxr->rx_buffers[i];
4211                         if (rxbuf->buf != NULL) {
4212                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4213                                     BUS_DMASYNC_POSTREAD);
4214                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4215                                 rxbuf->buf->m_flags |= M_PKTHDR;
4216                                 m_freem(rxbuf->buf);
4217                         }
4218                         rxbuf->buf = NULL;
4219                         if (rxbuf->pmap != NULL) {
4220                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4221                                 rxbuf->pmap = NULL;
4222                         }
4223                 }
4224                 if (rxr->rx_buffers != NULL) {
4225                         free(rxr->rx_buffers, M_DEVBUF);
4226                         rxr->rx_buffers = NULL;
4227                 }
4228         }
4229
4230         if (rxr->ptag != NULL) {
4231                 bus_dma_tag_destroy(rxr->ptag);
4232                 rxr->ptag = NULL;
4233         }
4234
4235         return;
4236 }
4237
4238 static __inline void
4239 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4240 {
4241
4242         /*
4243          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4244          * should be computed by hardware. Also it should not have VLAN tag in
4245          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4246          */
4247         if (rxr->lro_enabled &&
4248             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4249             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4250             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4251             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4252             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4253             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4254             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4255             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4256                 /*
4257                  * Send to the stack if:
4258                  **  - LRO not enabled, or
4259                  **  - no LRO resources, or
4260                  **  - lro enqueue fails
4261                  */
4262                 if (rxr->lro.lro_cnt != 0)
4263                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4264                                 return;
4265         }
4266         IXGBE_RX_UNLOCK(rxr);
4267         (*ifp->if_input)(ifp, m);
4268         IXGBE_RX_LOCK(rxr);
4269 }
4270
4271 static __inline void
4272 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4273 {
4274         struct ixgbe_rx_buf     *rbuf;
4275
4276         rbuf = &rxr->rx_buffers[i];
4277
4278         if (rbuf->fmp != NULL) {/* Partial chain ? */
4279                 rbuf->fmp->m_flags |= M_PKTHDR;
4280                 m_freem(rbuf->fmp);
4281                 rbuf->fmp = NULL;
4282         }
4283
4284         /*
4285         ** With advanced descriptors the writeback
4286         ** clobbers the buffer addrs, so its easier
4287         ** to just free the existing mbufs and take
4288         ** the normal refresh path to get new buffers
4289         ** and mapping.
4290         */
4291         if (rbuf->buf) {
4292                 m_free(rbuf->buf);
4293                 rbuf->buf = NULL;
4294         }
4295
4296         return;
4297 }
4298
4299
4300 /*********************************************************************
4301  *
4302  *  This routine executes in interrupt context. It replenishes
4303  *  the mbufs in the descriptor and sends data which has been
4304  *  dma'ed into host memory to upper layer.
4305  *
4306  *  We loop at most count times if count is > 0, or until done if
4307  *  count < 0.
4308  *
4309  *  Return TRUE for more work, FALSE for all clean.
4310  *********************************************************************/
4311 static bool
4312 ixgbe_rxeof(struct ix_queue *que)
4313 {
4314         struct adapter          *adapter = que->adapter;
4315         struct rx_ring          *rxr = que->rxr;
4316         struct ifnet            *ifp = adapter->ifp;
4317         struct lro_ctrl         *lro = &rxr->lro;
4318         struct lro_entry        *queued;
4319         int                     i, nextp, processed = 0;
4320         u32                     staterr = 0;
4321         u16                     count = rxr->process_limit;
4322         union ixgbe_adv_rx_desc *cur;
4323         struct ixgbe_rx_buf     *rbuf, *nbuf;
4324
4325         IXGBE_RX_LOCK(rxr);
4326
4327 #ifdef DEV_NETMAP
4328         if (ifp->if_capenable & IFCAP_NETMAP) {
4329                 /*
4330                  * Same as the txeof routine: only wakeup clients on intr.
4331                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4332                  * mitigation (ixgbe_rxsync() will not look for new packets
4333                  * unless NKR_PENDINTR is set).
4334                  */
4335                 struct netmap_adapter *na = NA(ifp);
4336
4337                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4338                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4339                 IXGBE_RX_UNLOCK(rxr);
4340                 IXGBE_CORE_LOCK(adapter);
4341                 selwakeuppri(&na->rx_si, PI_NET);
4342                 IXGBE_CORE_UNLOCK(adapter);
4343                 return (FALSE);
4344         }
4345 #endif /* DEV_NETMAP */
4346         for (i = rxr->next_to_check; count != 0;) {
4347                 struct mbuf     *sendmp, *mp;
4348                 u32             rsc, ptype;
4349                 u16             len;
4350                 u16             vtag = 0;
4351                 bool            eop;
4352
4353                 /* Sync the ring. */
4354                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4355                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4356
4357                 cur = &rxr->rx_base[i];
4358                 staterr = le32toh(cur->wb.upper.status_error);
4359
4360                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4361                         break;
4362                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4363                         break;
4364
4365                 count--;
4366                 sendmp = NULL;
4367                 nbuf = NULL;
4368                 rsc = 0;
4369                 cur->wb.upper.status_error = 0;
4370                 rbuf = &rxr->rx_buffers[i];
4371                 mp = rbuf->buf;
4372
4373                 len = le16toh(cur->wb.upper.length);
4374                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4375                     IXGBE_RXDADV_PKTTYPE_MASK;
4376                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4377
4378                 /* Make sure bad packets are discarded */
4379                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4380                     (rxr->discard)) {
4381                         ifp->if_ierrors++;
4382                         rxr->rx_discarded++;
4383                         if (eop)
4384                                 rxr->discard = FALSE;
4385                         else
4386                                 rxr->discard = TRUE;
4387                         ixgbe_rx_discard(rxr, i);
4388                         goto next_desc;
4389                 }
4390
4391                 /*
4392                 ** On 82599 which supports a hardware
4393                 ** LRO (called HW RSC), packets need
4394                 ** not be fragmented across sequential
4395                 ** descriptors, rather the next descriptor
4396                 ** is indicated in bits of the descriptor.
4397                 ** This also means that we might proceses
4398                 ** more than one packet at a time, something
4399                 ** that has never been true before, it
4400                 ** required eliminating global chain pointers
4401                 ** in favor of what we are doing here.  -jfv
4402                 */
4403                 if (!eop) {
4404                         /*
4405                         ** Figure out the next descriptor
4406                         ** of this frame.
4407                         */
4408                         if (rxr->hw_rsc == TRUE) {
4409                                 rsc = ixgbe_rsc_count(cur);
4410                                 rxr->rsc_num += (rsc - 1);
4411                         }
4412                         if (rsc) { /* Get hardware index */
4413                                 nextp = ((staterr &
4414                                     IXGBE_RXDADV_NEXTP_MASK) >>
4415                                     IXGBE_RXDADV_NEXTP_SHIFT);
4416                         } else { /* Just sequential */
4417                                 nextp = i + 1;
4418                                 if (nextp == adapter->num_rx_desc)
4419                                         nextp = 0;
4420                         }
4421                         nbuf = &rxr->rx_buffers[nextp];
4422                         prefetch(nbuf);
4423                 }
4424                 /*
4425                 ** Rather than using the fmp/lmp global pointers
4426                 ** we now keep the head of a packet chain in the
4427                 ** buffer struct and pass this along from one
4428                 ** descriptor to the next, until we get EOP.
4429                 */
4430                 mp->m_len = len;
4431                 /*
4432                 ** See if there is a stored head
4433                 ** that determines what we are
4434                 */
4435                 sendmp = rbuf->fmp;
4436                 if (sendmp != NULL) {  /* secondary frag */
4437                         rbuf->buf = rbuf->fmp = NULL;
4438                         mp->m_flags &= ~M_PKTHDR;
4439                         sendmp->m_pkthdr.len += mp->m_len;
4440                 } else {
4441                         /*
4442                          * Optimize.  This might be a small packet,
4443                          * maybe just a TCP ACK.  Do a fast copy that
4444                          * is cache aligned into a new mbuf, and
4445                          * leave the old mbuf+cluster for re-use.
4446                          */
4447                         if (eop && len <= IXGBE_RX_COPY_LEN) {
4448                                 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4449                                 if (sendmp != NULL) {
4450                                         sendmp->m_data +=
4451                                             IXGBE_RX_COPY_ALIGN;
4452                                         ixgbe_bcopy(mp->m_data,
4453                                             sendmp->m_data, len);
4454                                         sendmp->m_len = len;
4455                                         rxr->rx_copies++;
4456                                         rbuf->flags |= IXGBE_RX_COPY;
4457                                 }
4458                         }
4459                         if (sendmp == NULL) {
4460                                 rbuf->buf = rbuf->fmp = NULL;
4461                                 sendmp = mp;
4462                         }
4463
4464                         /* first desc of a non-ps chain */
4465                         sendmp->m_flags |= M_PKTHDR;
4466                         sendmp->m_pkthdr.len = mp->m_len;
4467                 }
4468                 ++processed;
4469
4470                 /* Pass the head pointer on */
4471                 if (eop == 0) {
4472                         nbuf->fmp = sendmp;
4473                         sendmp = NULL;
4474                         mp->m_next = nbuf->buf;
4475                 } else { /* Sending this frame */
4476                         sendmp->m_pkthdr.rcvif = ifp;
4477                         ifp->if_ipackets++;
4478                         rxr->rx_packets++;
4479                         /* capture data for AIM */
4480                         rxr->bytes += sendmp->m_pkthdr.len;
4481                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4482                         /* Process vlan info */
4483                         if ((rxr->vtag_strip) &&
4484                             (staterr & IXGBE_RXD_STAT_VP))
4485                                 vtag = le16toh(cur->wb.upper.vlan);
4486                         if (vtag) {
4487                                 sendmp->m_pkthdr.ether_vtag = vtag;
4488                                 sendmp->m_flags |= M_VLANTAG;
4489                         }
4490                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4491                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4492 #if __FreeBSD_version >= 800000
4493                         sendmp->m_pkthdr.flowid = que->msix;
4494                         sendmp->m_flags |= M_FLOWID;
4495 #endif
4496                 }
4497 next_desc:
4498                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4499                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4500
4501                 /* Advance our pointers to the next descriptor. */
4502                 if (++i == rxr->num_desc)
4503                         i = 0;
4504
4505                 /* Now send to the stack or do LRO */
4506                 if (sendmp != NULL) {
4507                         rxr->next_to_check = i;
4508                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4509                         i = rxr->next_to_check;
4510                 }
4511
4512                /* Every 8 descriptors we go to refresh mbufs */
4513                 if (processed == 8) {
4514                         ixgbe_refresh_mbufs(rxr, i);
4515                         processed = 0;
4516                 }
4517         }
4518
4519         /* Refresh any remaining buf structs */
4520         if (ixgbe_rx_unrefreshed(rxr))
4521                 ixgbe_refresh_mbufs(rxr, i);
4522
4523         rxr->next_to_check = i;
4524
4525         /*
4526          * Flush any outstanding LRO work
4527          */
4528         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4529                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4530                 tcp_lro_flush(lro, queued);
4531         }
4532
4533         IXGBE_RX_UNLOCK(rxr);
4534
4535         /*
4536         ** We still have cleaning to do?
4537         ** Schedule another interrupt if so.
4538         */
4539         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4540                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4541                 return (TRUE);
4542         }
4543
4544         return (FALSE);
4545 }
4546
4547
4548 /*********************************************************************
4549  *
4550  *  Verify that the hardware indicated that the checksum is valid.
4551  *  Inform the stack about the status of checksum so that stack
4552  *  doesn't spend time verifying the checksum.
4553  *
4554  *********************************************************************/
4555 static void
4556 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4557 {
4558         u16     status = (u16) staterr;
4559         u8      errors = (u8) (staterr >> 24);
4560         bool    sctp = FALSE;
4561
4562         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4563             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4564                 sctp = TRUE;
4565
4566         if (status & IXGBE_RXD_STAT_IPCS) {
4567                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4568                         /* IP Checksum Good */
4569                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4570                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4571
4572                 } else
4573                         mp->m_pkthdr.csum_flags = 0;
4574         }
4575         if (status & IXGBE_RXD_STAT_L4CS) {
4576                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4577 #if __FreeBSD_version >= 800000
4578                 if (sctp)
4579                         type = CSUM_SCTP_VALID;
4580 #endif
4581                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4582                         mp->m_pkthdr.csum_flags |= type;
4583                         if (!sctp)
4584                                 mp->m_pkthdr.csum_data = htons(0xffff);
4585                 }
4586         }
4587         return;
4588 }
4589
4590
4591 /*
4592 ** This routine is run via an vlan config EVENT,
4593 ** it enables us to use the HW Filter table since
4594 ** we can get the vlan id. This just creates the
4595 ** entry in the soft version of the VFTA, init will
4596 ** repopulate the real table.
4597 */
4598 static void
4599 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4600 {
4601         struct adapter  *adapter = ifp->if_softc;
4602         u16             index, bit;
4603
4604         if (ifp->if_softc !=  arg)   /* Not our event */
4605                 return;
4606
4607         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4608                 return;
4609
4610         IXGBE_CORE_LOCK(adapter);
4611         index = (vtag >> 5) & 0x7F;
4612         bit = vtag & 0x1F;
4613         adapter->shadow_vfta[index] |= (1 << bit);
4614         ++adapter->num_vlans;
4615         ixgbe_init_locked(adapter);
4616         IXGBE_CORE_UNLOCK(adapter);
4617 }
4618
4619 /*
4620 ** This routine is run via an vlan
4621 ** unconfig EVENT, remove our entry
4622 ** in the soft vfta.
4623 */
4624 static void
4625 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4626 {
4627         struct adapter  *adapter = ifp->if_softc;
4628         u16             index, bit;
4629
4630         if (ifp->if_softc !=  arg)
4631                 return;
4632
4633         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4634                 return;
4635
4636         IXGBE_CORE_LOCK(adapter);
4637         index = (vtag >> 5) & 0x7F;
4638         bit = vtag & 0x1F;
4639         adapter->shadow_vfta[index] &= ~(1 << bit);
4640         --adapter->num_vlans;
4641         /* Re-init to load the changes */
4642         ixgbe_init_locked(adapter);
4643         IXGBE_CORE_UNLOCK(adapter);
4644 }
4645
4646 static void
4647 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4648 {
4649         struct ifnet    *ifp = adapter->ifp;
4650         struct ixgbe_hw *hw = &adapter->hw;
4651         struct rx_ring  *rxr;
4652         u32             ctrl;
4653
4654
4655         /*
4656         ** We get here thru init_locked, meaning
4657         ** a soft reset, this has already cleared
4658         ** the VFTA and other state, so if there
4659         ** have been no vlan's registered do nothing.
4660         */
4661         if (adapter->num_vlans == 0)
4662                 return;
4663
4664         /*
4665         ** A soft reset zero's out the VFTA, so
4666         ** we need to repopulate it now.
4667         */
4668         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4669                 if (adapter->shadow_vfta[i] != 0)
4670                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4671                             adapter->shadow_vfta[i]);
4672
4673         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4674         /* Enable the Filter Table if enabled */
4675         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4676                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4677                 ctrl |= IXGBE_VLNCTRL_VFE;
4678         }
4679         if (hw->mac.type == ixgbe_mac_82598EB)
4680                 ctrl |= IXGBE_VLNCTRL_VME;
4681         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4682
4683         /* Setup the queues for vlans */
4684         for (int i = 0; i < adapter->num_queues; i++) {
4685                 rxr = &adapter->rx_rings[i];
4686                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4687                 if (hw->mac.type != ixgbe_mac_82598EB) {
4688                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4689                         ctrl |= IXGBE_RXDCTL_VME;
4690                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4691                 }
4692                 rxr->vtag_strip = TRUE;
4693         }
4694 }
4695
4696 static void
4697 ixgbe_enable_intr(struct adapter *adapter)
4698 {
4699         struct ixgbe_hw *hw = &adapter->hw;
4700         struct ix_queue *que = adapter->queues;
4701         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4702
4703
4704         /* Enable Fan Failure detection */
4705         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4706                     mask |= IXGBE_EIMS_GPI_SDP1;
4707         else {
4708                     mask |= IXGBE_EIMS_ECC;
4709                     mask |= IXGBE_EIMS_GPI_SDP0;
4710                     mask |= IXGBE_EIMS_GPI_SDP1;
4711                     mask |= IXGBE_EIMS_GPI_SDP2;
4712 #ifdef IXGBE_FDIR
4713                     mask |= IXGBE_EIMS_FLOW_DIR;
4714 #endif
4715         }
4716
4717         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4718
4719         /* With RSS we use auto clear */
4720         if (adapter->msix_mem) {
4721                 mask = IXGBE_EIMS_ENABLE_MASK;
4722                 /* Don't autoclear Link */
4723                 mask &= ~IXGBE_EIMS_OTHER;
4724                 mask &= ~IXGBE_EIMS_LSC;
4725                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4726         }
4727
4728         /*
4729         ** Now enable all queues, this is done separately to
4730         ** allow for handling the extended (beyond 32) MSIX
4731         ** vectors that can be used by 82599
4732         */
4733         for (int i = 0; i < adapter->num_queues; i++, que++)
4734                 ixgbe_enable_queue(adapter, que->msix);
4735
4736         IXGBE_WRITE_FLUSH(hw);
4737
4738         return;
4739 }
4740
4741 static void
4742 ixgbe_disable_intr(struct adapter *adapter)
4743 {
4744         if (adapter->msix_mem)
4745                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4746         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4747                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4748         } else {
4749                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4750                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4751                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4752         }
4753         IXGBE_WRITE_FLUSH(&adapter->hw);
4754         return;
4755 }
4756
4757 u16
4758 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4759 {
4760         u16 value;
4761
4762         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4763             reg, 2);
4764
4765         return (value);
4766 }
4767
4768 void
4769 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4770 {
4771         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4772             reg, value, 2);
4773
4774         return;
4775 }
4776
4777 /*
4778 ** Setup the correct IVAR register for a particular MSIX interrupt
4779 **   (yes this is all very magic and confusing :)
4780 **  - entry is the register array entry
4781 **  - vector is the MSIX vector for this queue
4782 **  - type is RX/TX/MISC
4783 */
4784 static void
4785 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4786 {
4787         struct ixgbe_hw *hw = &adapter->hw;
4788         u32 ivar, index;
4789
4790         vector |= IXGBE_IVAR_ALLOC_VAL;
4791
4792         switch (hw->mac.type) {
4793
4794         case ixgbe_mac_82598EB:
4795                 if (type == -1)
4796                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4797                 else
4798                         entry += (type * 64);
4799                 index = (entry >> 2) & 0x1F;
4800                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4801                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4802                 ivar |= (vector << (8 * (entry & 0x3)));
4803                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4804                 break;
4805
4806         case ixgbe_mac_82599EB:
4807         case ixgbe_mac_X540:
4808                 if (type == -1) { /* MISC IVAR */
4809                         index = (entry & 1) * 8;
4810                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4811                         ivar &= ~(0xFF << index);
4812                         ivar |= (vector << index);
4813                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4814                 } else {        /* RX/TX IVARS */
4815                         index = (16 * (entry & 1)) + (8 * type);
4816                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4817                         ivar &= ~(0xFF << index);
4818                         ivar |= (vector << index);
4819                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4820                 }
4821
4822         default:
4823                 break;
4824         }
4825 }
4826
4827 static void
4828 ixgbe_configure_ivars(struct adapter *adapter)
4829 {
4830         struct  ix_queue *que = adapter->queues;
4831         u32 newitr;
4832
4833         if (ixgbe_max_interrupt_rate > 0)
4834                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
4835         else
4836                 newitr = 0;
4837
4838         for (int i = 0; i < adapter->num_queues; i++, que++) {
4839                 /* First the RX queue entry */
4840                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4841                 /* ... and the TX */
4842                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4843                 /* Set an Initial EITR value */
4844                 IXGBE_WRITE_REG(&adapter->hw,
4845                     IXGBE_EITR(que->msix), newitr);
4846         }
4847
4848         /* For the Link interrupt */
4849         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4850 }
4851
4852 /*
4853 ** ixgbe_sfp_probe - called in the local timer to
4854 ** determine if a port had optics inserted.
4855 */
4856 static bool ixgbe_sfp_probe(struct adapter *adapter)
4857 {
4858         struct ixgbe_hw *hw = &adapter->hw;
4859         device_t        dev = adapter->dev;
4860         bool            result = FALSE;
4861
4862         if ((hw->phy.type == ixgbe_phy_nl) &&
4863             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4864                 s32 ret = hw->phy.ops.identify_sfp(hw);
4865                 if (ret)
4866                         goto out;
4867                 ret = hw->phy.ops.reset(hw);
4868                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4869                         device_printf(dev,"Unsupported SFP+ module detected!");
4870                         printf(" Reload driver with supported module.\n");
4871                         adapter->sfp_probe = FALSE;
4872                         goto out;
4873                 } else
4874                         device_printf(dev,"SFP+ module detected!\n");
4875                 /* We now have supported optics */
4876                 adapter->sfp_probe = FALSE;
4877                 /* Set the optics type so system reports correctly */
4878                 ixgbe_setup_optics(adapter);
4879                 result = TRUE;
4880         }
4881 out:
4882         return (result);
4883 }
4884
4885 /*
4886 ** Tasklet handler for MSIX Link interrupts
4887 **  - do outside interrupt since it might sleep
4888 */
4889 static void
4890 ixgbe_handle_link(void *context, int pending)
4891 {
4892         struct adapter  *adapter = context;
4893
4894         ixgbe_check_link(&adapter->hw,
4895             &adapter->link_speed, &adapter->link_up, 0);
4896         ixgbe_update_link_status(adapter);
4897 }
4898
4899 /*
4900 ** Tasklet for handling SFP module interrupts
4901 */
4902 static void
4903 ixgbe_handle_mod(void *context, int pending)
4904 {
4905         struct adapter  *adapter = context;
4906         struct ixgbe_hw *hw = &adapter->hw;
4907         device_t        dev = adapter->dev;
4908         u32 err;
4909
4910         err = hw->phy.ops.identify_sfp(hw);
4911         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4912                 device_printf(dev,
4913                     "Unsupported SFP+ module type was detected.\n");
4914                 return;
4915         }
4916         err = hw->mac.ops.setup_sfp(hw);
4917         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4918                 device_printf(dev,
4919                     "Setup failure - unsupported SFP+ module type.\n");
4920                 return;
4921         }
4922         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4923         return;
4924 }
4925
4926
4927 /*
4928 ** Tasklet for handling MSF (multispeed fiber) interrupts
4929 */
4930 static void
4931 ixgbe_handle_msf(void *context, int pending)
4932 {
4933         struct adapter  *adapter = context;
4934         struct ixgbe_hw *hw = &adapter->hw;
4935         u32 autoneg;
4936         bool negotiate;
4937
4938         autoneg = hw->phy.autoneg_advertised;
4939         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4940                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4941         if (hw->mac.ops.setup_link)
4942                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4943         return;
4944 }
4945
4946 #ifdef IXGBE_FDIR
4947 /*
4948 ** Tasklet for reinitializing the Flow Director filter table
4949 */
4950 static void
4951 ixgbe_reinit_fdir(void *context, int pending)
4952 {
4953         struct adapter  *adapter = context;
4954         struct ifnet   *ifp = adapter->ifp;
4955
4956         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4957                 return;
4958         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4959         adapter->fdir_reinit = 0;
4960         /* re-enable flow director interrupts */
4961         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4962         /* Restart the interface */
4963         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4964         return;
4965 }
4966 #endif
4967
4968 /**********************************************************************
4969  *
4970  *  Update the board statistics counters.
4971  *
4972  **********************************************************************/
4973 static void
4974 ixgbe_update_stats_counters(struct adapter *adapter)
4975 {
4976         struct ifnet   *ifp = adapter->ifp;
4977         struct ixgbe_hw *hw = &adapter->hw;
4978         u32  missed_rx = 0, bprc, lxon, lxoff, total;
4979         u64  total_missed_rx = 0;
4980
4981         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
4982         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
4983         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
4984         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
4985
4986         for (int i = 0; i < 8; i++) {
4987                 u32 mp;
4988                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
4989                 /* missed_rx tallies misses for the gprc workaround */
4990                 missed_rx += mp;
4991                 /* global total per queue */
4992                 adapter->stats.mpc[i] += mp;
4993                 /* Running comprehensive total for stats display */
4994                 total_missed_rx += adapter->stats.mpc[i];
4995                 if (hw->mac.type == ixgbe_mac_82598EB)
4996                         adapter->stats.rnbc[i] +=
4997                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
4998                 adapter->stats.pxontxc[i] +=
4999                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5000                 adapter->stats.pxonrxc[i] +=
5001                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5002                 adapter->stats.pxofftxc[i] +=
5003                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5004                 adapter->stats.pxoffrxc[i] +=
5005                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5006                 adapter->stats.pxon2offc[i] +=
5007                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5008         }
5009         for (int i = 0; i < 16; i++) {
5010                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5011                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5012                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5013                 adapter->stats.qbrc[i] +=
5014                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5015                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5016                 adapter->stats.qbtc[i] +=
5017                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5018                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5019         }
5020         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5021         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5022         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5023
5024         /* Hardware workaround, gprc counts missed packets */
5025         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5026         adapter->stats.gprc -= missed_rx;
5027
5028         if (hw->mac.type != ixgbe_mac_82598EB) {
5029                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5030                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5031                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5032                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5033                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5034                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5035                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5036                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5037         } else {
5038                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5039                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5040                 /* 82598 only has a counter in the high register */
5041                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5042                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5043                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5044         }
5045
5046         /*
5047          * Workaround: mprc hardware is incorrectly counting
5048          * broadcasts, so for now we subtract those.
5049          */
5050         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5051         adapter->stats.bprc += bprc;
5052         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5053         if (hw->mac.type == ixgbe_mac_82598EB)
5054                 adapter->stats.mprc -= bprc;
5055
5056         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5057         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5058         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5059         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5060         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5061         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5062
5063         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5064         adapter->stats.lxontxc += lxon;
5065         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5066         adapter->stats.lxofftxc += lxoff;
5067         total = lxon + lxoff;
5068
5069         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5070         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5071         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5072         adapter->stats.gptc -= total;
5073         adapter->stats.mptc -= total;
5074         adapter->stats.ptc64 -= total;
5075         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5076
5077         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5078         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5079         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5080         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5081         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5082         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5083         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5084         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5085         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5086         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5087         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5088         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5089         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5090         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5091         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5092         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5093         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5094         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5095         /* Only read FCOE on 82599 */
5096         if (hw->mac.type != ixgbe_mac_82598EB) {
5097                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5098                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5099                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5100                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5101                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5102         }
5103
5104         /* Fill out the OS statistics structure */
5105         ifp->if_ipackets = adapter->stats.gprc;
5106         ifp->if_opackets = adapter->stats.gptc;
5107         ifp->if_ibytes = adapter->stats.gorc;
5108         ifp->if_obytes = adapter->stats.gotc;
5109         ifp->if_imcasts = adapter->stats.mprc;
5110         ifp->if_omcasts = adapter->stats.mptc;
5111         ifp->if_collisions = 0;
5112
5113         /* Rx Errors */
5114         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5115                 adapter->stats.rlec;
5116 }
5117
5118 /** ixgbe_sysctl_tdh_handler - Handler function
5119  *  Retrieves the TDH value from the hardware
5120  */
5121 static int
5122 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5123 {
5124         int error;
5125
5126         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5127         if (!txr) return 0;
5128
5129         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5130         error = sysctl_handle_int(oidp, &val, 0, req);
5131         if (error || !req->newptr)
5132                 return error;
5133         return 0;
5134 }
5135
5136 /** ixgbe_sysctl_tdt_handler - Handler function
5137  *  Retrieves the TDT value from the hardware
5138  */
5139 static int
5140 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5141 {
5142         int error;
5143
5144         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5145         if (!txr) return 0;
5146
5147         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5148         error = sysctl_handle_int(oidp, &val, 0, req);
5149         if (error || !req->newptr)
5150                 return error;
5151         return 0;
5152 }
5153
5154 /** ixgbe_sysctl_rdh_handler - Handler function
5155  *  Retrieves the RDH value from the hardware
5156  */
5157 static int
5158 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5159 {
5160         int error;
5161
5162         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5163         if (!rxr) return 0;
5164
5165         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5166         error = sysctl_handle_int(oidp, &val, 0, req);
5167         if (error || !req->newptr)
5168                 return error;
5169         return 0;
5170 }
5171
5172 /** ixgbe_sysctl_rdt_handler - Handler function
5173  *  Retrieves the RDT value from the hardware
5174  */
5175 static int
5176 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5177 {
5178         int error;
5179
5180         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5181         if (!rxr) return 0;
5182
5183         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5184         error = sysctl_handle_int(oidp, &val, 0, req);
5185         if (error || !req->newptr)
5186                 return error;
5187         return 0;
5188 }
5189
5190 static int
5191 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5192 {
5193         int error;
5194         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5195         unsigned int reg, usec, rate;
5196
5197         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5198         usec = ((reg & 0x0FF8) >> 3);
5199         if (usec > 0)
5200                 rate = 500000 / usec;
5201         else
5202                 rate = 0;
5203         error = sysctl_handle_int(oidp, &rate, 0, req);
5204         if (error || !req->newptr)
5205                 return error;
5206         reg &= ~0xfff; /* default, no limitation */
5207         ixgbe_max_interrupt_rate = 0;
5208         if (rate > 0 && rate < 500000) {
5209                 if (rate < 1000)
5210                         rate = 1000;
5211                 ixgbe_max_interrupt_rate = rate;
5212                 reg |= ((4000000/rate) & 0xff8 );
5213         }
5214         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5215         return 0;
5216 }
5217
5218 /*
5219  * Add sysctl variables, one per statistic, to the system.
5220  */
5221 static void
5222 ixgbe_add_hw_stats(struct adapter *adapter)
5223 {
5224
5225         device_t dev = adapter->dev;
5226
5227         struct tx_ring *txr = adapter->tx_rings;
5228         struct rx_ring *rxr = adapter->rx_rings;
5229
5230         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5231         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5232         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5233         struct ixgbe_hw_stats *stats = &adapter->stats;
5234
5235         struct sysctl_oid *stat_node, *queue_node;
5236         struct sysctl_oid_list *stat_list, *queue_list;
5237
5238 #define QUEUE_NAME_LEN 32
5239         char namebuf[QUEUE_NAME_LEN];
5240
5241         /* Driver Statistics */
5242         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5243                         CTLFLAG_RD, &adapter->dropped_pkts,
5244                         "Driver dropped packets");
5245         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5246                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5247                         "m_defrag() failed");
5248         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5249                         CTLFLAG_RD, &adapter->watchdog_events,
5250                         "Watchdog timeouts");
5251         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5252                         CTLFLAG_RD, &adapter->link_irq,
5253                         "Link MSIX IRQ Handled");
5254
5255         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5256                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5257                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5258                                             CTLFLAG_RD, NULL, "Queue Name");
5259                 queue_list = SYSCTL_CHILDREN(queue_node);
5260
5261                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5262                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5263                                 sizeof(&adapter->queues[i]),
5264                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5265                                 "Interrupt Rate");
5266                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5267                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
5268                                 "irqs on this queue");
5269                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5270                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5271                                 ixgbe_sysctl_tdh_handler, "IU",
5272                                 "Transmit Descriptor Head");
5273                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5274                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5275                                 ixgbe_sysctl_tdt_handler, "IU",
5276                                 "Transmit Descriptor Tail");
5277                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5278                                 CTLFLAG_RD, &txr->tso_tx,
5279                                 "TSO");
5280                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5281                                 CTLFLAG_RD, &txr->no_tx_dma_setup,
5282                                 "Driver tx dma failure in xmit");
5283                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5284                                 CTLFLAG_RD, &txr->no_desc_avail,
5285                                 "Queue No Descriptor Available");
5286                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5287                                 CTLFLAG_RD, &txr->total_packets,
5288                                 "Queue Packets Transmitted");
5289         }
5290
5291         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5292                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5293                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5294                                             CTLFLAG_RD, NULL, "Queue Name");
5295                 queue_list = SYSCTL_CHILDREN(queue_node);
5296
5297                 struct lro_ctrl *lro = &rxr->lro;
5298
5299                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5300                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5301                                             CTLFLAG_RD, NULL, "Queue Name");
5302                 queue_list = SYSCTL_CHILDREN(queue_node);
5303
5304                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5305                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5306                                 ixgbe_sysctl_rdh_handler, "IU",
5307                                 "Receive Descriptor Head");
5308                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5309                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5310                                 ixgbe_sysctl_rdt_handler, "IU",
5311                                 "Receive Descriptor Tail");
5312                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5313                                 CTLFLAG_RD, &rxr->rx_packets,
5314                                 "Queue Packets Received");
5315                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5316                                 CTLFLAG_RD, &rxr->rx_bytes,
5317                                 "Queue Bytes Received");
5318                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5319                                 CTLFLAG_RD, &rxr->rx_copies,
5320                                 "Copied RX Frames");
5321                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5322                                 CTLFLAG_RD, &lro->lro_queued, 0,
5323                                 "LRO Queued");
5324                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5325                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5326                                 "LRO Flushed");
5327         }
5328
5329         /* MAC stats get the own sub node */
5330
5331         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5332                                     CTLFLAG_RD, NULL, "MAC Statistics");
5333         stat_list = SYSCTL_CHILDREN(stat_node);
5334
5335         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5336                         CTLFLAG_RD, &stats->crcerrs,
5337                         "CRC Errors");
5338         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5339                         CTLFLAG_RD, &stats->illerrc,
5340                         "Illegal Byte Errors");
5341         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5342                         CTLFLAG_RD, &stats->errbc,
5343                         "Byte Errors");
5344         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5345                         CTLFLAG_RD, &stats->mspdc,
5346                         "MAC Short Packets Discarded");
5347         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5348                         CTLFLAG_RD, &stats->mlfc,
5349                         "MAC Local Faults");
5350         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5351                         CTLFLAG_RD, &stats->mrfc,
5352                         "MAC Remote Faults");
5353         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5354                         CTLFLAG_RD, &stats->rlec,
5355                         "Receive Length Errors");
5356
5357         /* Flow Control stats */
5358         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5359                         CTLFLAG_RD, &stats->lxontxc,
5360                         "Link XON Transmitted");
5361         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5362                         CTLFLAG_RD, &stats->lxonrxc,
5363                         "Link XON Received");
5364         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5365                         CTLFLAG_RD, &stats->lxofftxc,
5366                         "Link XOFF Transmitted");
5367         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5368                         CTLFLAG_RD, &stats->lxoffrxc,
5369                         "Link XOFF Received");
5370
5371         /* Packet Reception Stats */
5372         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5373                         CTLFLAG_RD, &stats->tor,
5374                         "Total Octets Received");
5375         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5376                         CTLFLAG_RD, &stats->gorc,
5377                         "Good Octets Received");
5378         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5379                         CTLFLAG_RD, &stats->tpr,
5380                         "Total Packets Received");
5381         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5382                         CTLFLAG_RD, &stats->gprc,
5383                         "Good Packets Received");
5384         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5385                         CTLFLAG_RD, &stats->mprc,
5386                         "Multicast Packets Received");
5387         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5388                         CTLFLAG_RD, &stats->bprc,
5389                         "Broadcast Packets Received");
5390         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5391                         CTLFLAG_RD, &stats->prc64,
5392                         "64 byte frames received ");
5393         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5394                         CTLFLAG_RD, &stats->prc127,
5395                         "65-127 byte frames received");
5396         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5397                         CTLFLAG_RD, &stats->prc255,
5398                         "128-255 byte frames received");
5399         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5400                         CTLFLAG_RD, &stats->prc511,
5401                         "256-511 byte frames received");
5402         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5403                         CTLFLAG_RD, &stats->prc1023,
5404                         "512-1023 byte frames received");
5405         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5406                         CTLFLAG_RD, &stats->prc1522,
5407                         "1023-1522 byte frames received");
5408         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5409                         CTLFLAG_RD, &stats->ruc,
5410                         "Receive Undersized");
5411         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5412                         CTLFLAG_RD, &stats->rfc,
5413                         "Fragmented Packets Received ");
5414         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5415                         CTLFLAG_RD, &stats->roc,
5416                         "Oversized Packets Received");
5417         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5418                         CTLFLAG_RD, &stats->rjc,
5419                         "Received Jabber");
5420         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5421                         CTLFLAG_RD, &stats->mngprc,
5422                         "Management Packets Received");
5423         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5424                         CTLFLAG_RD, &stats->mngptc,
5425                         "Management Packets Dropped");
5426         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5427                         CTLFLAG_RD, &stats->xec,
5428                         "Checksum Errors");
5429
5430         /* Packet Transmission Stats */
5431         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5432                         CTLFLAG_RD, &stats->gotc,
5433                         "Good Octets Transmitted");
5434         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5435                         CTLFLAG_RD, &stats->tpt,
5436                         "Total Packets Transmitted");
5437         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5438                         CTLFLAG_RD, &stats->gptc,
5439                         "Good Packets Transmitted");
5440         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5441                         CTLFLAG_RD, &stats->bptc,
5442                         "Broadcast Packets Transmitted");
5443         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5444                         CTLFLAG_RD, &stats->mptc,
5445                         "Multicast Packets Transmitted");
5446         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5447                         CTLFLAG_RD, &stats->mngptc,
5448                         "Management Packets Transmitted");
5449         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5450                         CTLFLAG_RD, &stats->ptc64,
5451                         "64 byte frames transmitted ");
5452         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5453                         CTLFLAG_RD, &stats->ptc127,
5454                         "65-127 byte frames transmitted");
5455         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5456                         CTLFLAG_RD, &stats->ptc255,
5457                         "128-255 byte frames transmitted");
5458         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5459                         CTLFLAG_RD, &stats->ptc511,
5460                         "256-511 byte frames transmitted");
5461         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5462                         CTLFLAG_RD, &stats->ptc1023,
5463                         "512-1023 byte frames transmitted");
5464         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5465                         CTLFLAG_RD, &stats->ptc1522,
5466                         "1024-1522 byte frames transmitted");
5467 }
5468
5469 /*
5470 ** Set flow control using sysctl:
5471 ** Flow control values:
5472 **      0 - off
5473 **      1 - rx pause
5474 **      2 - tx pause
5475 **      3 - full
5476 */
5477 static int
5478 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5479 {
5480         int error, last;
5481         struct adapter *adapter = (struct adapter *) arg1;
5482
5483         last = adapter->fc;
5484         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5485         if ((error) || (req->newptr == NULL))
5486                 return (error);
5487
5488         /* Don't bother if it's not changed */
5489         if (adapter->fc == last)
5490                 return (0);
5491
5492         switch (adapter->fc) {
5493                 case ixgbe_fc_rx_pause:
5494                 case ixgbe_fc_tx_pause:
5495                 case ixgbe_fc_full:
5496                         adapter->hw.fc.requested_mode = adapter->fc;
5497                         if (adapter->num_queues > 1)
5498                                 ixgbe_disable_rx_drop(adapter);
5499                         break;
5500                 case ixgbe_fc_none:
5501                 default:
5502                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5503                         if (adapter->num_queues > 1)
5504                                 ixgbe_enable_rx_drop(adapter);
5505         }
5506         /* Don't autoneg if forcing a value */
5507         adapter->hw.fc.disable_fc_autoneg = TRUE;
5508         ixgbe_fc_enable(&adapter->hw);
5509         return error;
5510 }
5511
5512 /*
5513 ** Control link advertise speed:
5514 **      1 - advertise only 1G
5515 **      2 - advertise 100Mb
5516 **      3 - advertise normal
5517 */
5518 static int
5519 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5520 {
5521         int                     error = 0;
5522         struct adapter          *adapter;
5523         device_t                dev;
5524         struct ixgbe_hw         *hw;
5525         ixgbe_link_speed        speed, last;
5526
5527         adapter = (struct adapter *) arg1;
5528         dev = adapter->dev;
5529         hw = &adapter->hw;
5530         last = adapter->advertise;
5531
5532         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5533         if ((error) || (adapter->advertise == -1))
5534                 return (error);
5535
5536         if (adapter->advertise == last) /* no change */
5537                 return (0);
5538
5539         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5540             (hw->phy.multispeed_fiber)))
5541                 return (error);
5542
5543         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5544                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5545                 return (error);
5546         }
5547
5548         if (adapter->advertise == 1)
5549                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5550         else if (adapter->advertise == 2)
5551                 speed = IXGBE_LINK_SPEED_100_FULL;
5552         else if (adapter->advertise == 3)
5553                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5554                         IXGBE_LINK_SPEED_10GB_FULL;
5555         else /* bogus value */
5556                 return (error);
5557
5558         hw->mac.autotry_restart = TRUE;
5559         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5560
5561         return (error);
5562 }
5563
5564 /*
5565 ** Thermal Shutdown Trigger
5566 **   - cause a Thermal Overtemp IRQ
5567 */
5568 static int
5569 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5570 {
5571         int             error, fire = 0;
5572         struct adapter  *adapter = (struct adapter *) arg1;
5573         struct ixgbe_hw *hw = &adapter->hw;
5574
5575
5576         if (hw->mac.type != ixgbe_mac_X540)
5577                 return (0);
5578
5579         error = sysctl_handle_int(oidp, &fire, 0, req);
5580         if ((error) || (req->newptr == NULL))
5581                 return (error);
5582
5583         if (fire) {
5584                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5585                 reg |= IXGBE_EICR_TS;
5586                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5587         }
5588
5589         return (0);
5590 }
5591
5592 /*
5593 ** Enable the hardware to drop packets when the buffer is
5594 ** full. This is useful when multiqueue,so that no single
5595 ** queue being full stalls the entire RX engine. We only
5596 ** enable this when Multiqueue AND when Flow Control is
5597 ** disabled.
5598 */
5599 static void
5600 ixgbe_enable_rx_drop(struct adapter *adapter)
5601 {
5602         struct ixgbe_hw *hw = &adapter->hw;
5603
5604         for (int i = 0; i < adapter->num_queues; i++) {
5605                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5606                 srrctl |= IXGBE_SRRCTL_DROP_EN;
5607                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5608         }
5609 }
5610
5611 static void
5612 ixgbe_disable_rx_drop(struct adapter *adapter)
5613 {
5614         struct ixgbe_hw *hw = &adapter->hw;
5615
5616         for (int i = 0; i < adapter->num_queues; i++) {
5617                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5618                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5619                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5620         }
5621 }