sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2013, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #endif
  39
  40 #include "ixgbe.h"
  41
  42 /*********************************************************************
  43  *  Set this to one to display debug statistics
  44  *********************************************************************/
  45 int             ixgbe_display_debug_stats = 0;
  46
  47 /*********************************************************************
  48  *  Driver version
  49  *********************************************************************/
  50 char ixgbe_driver_version[] = "2.5.7 - STABLE/9";
  51
  52 /*********************************************************************
  53  *  PCI Device ID Table
  54  *
  55  *  Used by probe to select devices to load on
  56  *  Last field stores an index into ixgbe_strings
  57  *  Last entry must be all 0s
  58  *
  59  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  60  *********************************************************************/
  61
  62 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  63 {
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  86         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
  87         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  88         /* required last entry */
  89         {0, 0, 0, 0, 0}
  90 };
  91
  92 /*********************************************************************
  93  *  Table of branding strings
  94  *********************************************************************/
  95
  96 static char    *ixgbe_strings[] = {
  97         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  98 };
  99
 100 /*********************************************************************
 101  *  Function prototypes
 102  *********************************************************************/
 103 static int      ixgbe_probe(device_t);
 104 static int      ixgbe_attach(device_t);
 105 static int      ixgbe_detach(device_t);
 106 static int      ixgbe_shutdown(device_t);
 107 #ifdef IXGBE_LEGACY_TX
 108 static void     ixgbe_start(struct ifnet *);
 109 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 110 #else /* ! IXGBE_LEGACY_TX */
 111 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 112 static int      ixgbe_mq_start_locked(struct ifnet *,
 113                     struct tx_ring *, struct mbuf *);
 114 static void     ixgbe_qflush(struct ifnet *);
 115 static void     ixgbe_deferred_mq_start(void *, int);
 116 #endif /* IXGBE_LEGACY_TX */
 117 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 118 static void     ixgbe_init(void *);
 119 static void     ixgbe_init_locked(struct adapter *);
 120 static void     ixgbe_stop(void *);
 121 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 122 static int      ixgbe_media_change(struct ifnet *);
 123 static void     ixgbe_identify_hardware(struct adapter *);
 124 static int      ixgbe_allocate_pci_resources(struct adapter *);
 125 static int      ixgbe_allocate_msix(struct adapter *);
 126 static int      ixgbe_allocate_legacy(struct adapter *);
 127 static int      ixgbe_allocate_queues(struct adapter *);
 128 static int      ixgbe_setup_msix(struct adapter *);
 129 static void     ixgbe_free_pci_resources(struct adapter *);
 130 static void     ixgbe_local_timer(void *);
 131 static int      ixgbe_setup_interface(device_t, struct adapter *);
 132 static void     ixgbe_config_link(struct adapter *);
 133
 134 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 135 static int      ixgbe_setup_transmit_structures(struct adapter *);
 136 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 137 static void     ixgbe_initialize_transmit_units(struct adapter *);
 138 static void     ixgbe_free_transmit_structures(struct adapter *);
 139 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 140
 141 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 142 static int      ixgbe_setup_receive_structures(struct adapter *);
 143 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 144 static void     ixgbe_initialize_receive_units(struct adapter *);
 145 static void     ixgbe_free_receive_structures(struct adapter *);
 146 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 147 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 148
 149 static void     ixgbe_enable_intr(struct adapter *);
 150 static void     ixgbe_disable_intr(struct adapter *);
 151 static void     ixgbe_update_stats_counters(struct adapter *);
 152 static bool     ixgbe_txeof(struct tx_ring *);
 153 static bool     ixgbe_rxeof(struct ix_queue *);
 154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 155 static void     ixgbe_set_promisc(struct adapter *);
 156 static void     ixgbe_set_multi(struct adapter *);
 157 static void     ixgbe_update_link_status(struct adapter *);
 158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 164                     struct ixgbe_dma_alloc *, int);
 165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 166 static int      ixgbe_tx_ctx_setup(struct tx_ring *,
 167                     struct mbuf *, u32 *, u32 *);
 168 static int      ixgbe_tso_setup(struct tx_ring *,
 169                     struct mbuf *, u32 *, u32 *);
 170 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 171 static void     ixgbe_configure_ivars(struct adapter *);
 172 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 173
 174 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 175 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 176 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 177
 178 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 179
 180 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 181 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 182                     struct mbuf *, u32);
 183
 184 static void     ixgbe_enable_rx_drop(struct adapter *);
 185 static void     ixgbe_disable_rx_drop(struct adapter *);
 186
 187 /* Support for pluggable optic modules */
 188 static bool     ixgbe_sfp_probe(struct adapter *);
 189 static void     ixgbe_setup_optics(struct adapter *);
 190
 191 /* Legacy (single vector interrupt handler */
 192 static void     ixgbe_legacy_irq(void *);
 193
 194 /* The MSI/X Interrupt handlers */
 195 static void     ixgbe_msix_que(void *);
 196 static void     ixgbe_msix_link(void *);
 197
 198 /* Deferred interrupt tasklets */
 199 static void     ixgbe_handle_que(void *, int);
 200 static void     ixgbe_handle_link(void *, int);
 201 static void     ixgbe_handle_msf(void *, int);
 202 static void     ixgbe_handle_mod(void *, int);
 203
 204 #ifdef IXGBE_FDIR
 205 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 206 static void     ixgbe_reinit_fdir(void *, int);
 207 #endif
 208
 209 /*********************************************************************
 210  *  FreeBSD Device Interface Entry Points
 211  *********************************************************************/
 212
 213 static device_method_t ixgbe_methods[] = {
 214         /* Device interface */
 215         DEVMETHOD(device_probe, ixgbe_probe),
 216         DEVMETHOD(device_attach, ixgbe_attach),
 217         DEVMETHOD(device_detach, ixgbe_detach),
 218         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 219         DEVMETHOD_END
 220 };
 221
 222 static driver_t ixgbe_driver = {
 223         "ix", ixgbe_methods, sizeof(struct adapter),
 224 };
 225
 226 devclass_t ixgbe_devclass;
 227 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 228
 229 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 230 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 231
 232 /*
 233 ** TUNEABLE PARAMETERS:
 234 */
 235
 236 /*
 237 ** AIM: Adaptive Interrupt Moderation
 238 ** which means that the interrupt rate
 239 ** is varied over time based on the
 240 ** traffic for that interrupt vector
 241 */
 242 static int ixgbe_enable_aim = TRUE;
 243 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 244
 245 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 246 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 247
 248 /* How many packets rxeof tries to clean at a time */
 249 static int ixgbe_rx_process_limit = 256;
 250 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 251
 252 /* How many packets txeof tries to clean at a time */
 253 static int ixgbe_tx_process_limit = 256;
 254 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
 255
 256 /*
 257 ** Smart speed setting, default to on
 258 ** this only works as a compile option
 259 ** right now as its during attach, set
 260 ** this to 'ixgbe_smart_speed_off' to
 261 ** disable.
 262 */
 263 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 264
 265 /*
 266  * MSIX should be the default for best performance,
 267  * but this allows it to be forced off for testing.
 268  */
 269 static int ixgbe_enable_msix = 1;
 270 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 271
 272 /*
 273  * Number of Queues, can be set to 0,
 274  * it then autoconfigures based on the
 275  * number of cpus with a max of 8. This
 276  * can be overriden manually here.
 277  */
 278 static int ixgbe_num_queues = 0;
 279 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 280
 281 /*
 282 ** Number of TX descriptors per ring,
 283 ** setting higher than RX as this seems
 284 ** the better performing choice.
 285 */
 286 static int ixgbe_txd = PERFORM_TXD;
 287 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 288
 289 /* Number of RX descriptors per ring */
 290 static int ixgbe_rxd = PERFORM_RXD;
 291 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 292
 293 /*
 294 ** HW RSC control:
 295 **  this feature only works with
 296 **  IPv4, and only on 82599 and later.
 297 **  Also this will cause IP forwarding to
 298 **  fail and that can't be controlled by
 299 **  the stack as LRO can. For all these
 300 **  reasons I've deemed it best to leave
 301 **  this off and not bother with a tuneable
 302 **  interface, this would need to be compiled
 303 **  to enable.
 304 */
 305 static bool ixgbe_rsc_enable = FALSE;
 306
 307 /* Keep running tab on them for sanity check */
 308 static int ixgbe_total_ports;
 309
 310 #ifdef IXGBE_FDIR
 311 /*
 312 ** For Flow Director: this is the
 313 ** number of TX packets we sample
 314 ** for the filter pool, this means
 315 ** every 20th packet will be probed.
 316 **
 317 ** This feature can be disabled by
 318 ** setting this to 0.
 319 */
 320 static int atr_sample_rate = 20;
 321 /*
 322 ** Flow Director actually 'steals'
 323 ** part of the packet buffer as its
 324 ** filter pool, this variable controls
 325 ** how much it uses:
 326 **  0 = 64K, 1 = 128K, 2 = 256K
 327 */
 328 static int fdir_pballoc = 1;
 329 #endif
 330
 331 #ifdef DEV_NETMAP
 332 /*
 333  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 334  * be a reference on how to implement netmap support in a driver.
 335  * Additional comments are in ixgbe_netmap.h .
 336  *
 337  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 338  * that extend the standard driver.
 339  */
 340 #include <dev/netmap/ixgbe_netmap.h>
 341 #endif /* DEV_NETMAP */
 342
 343 /*********************************************************************
 344  *  Device identification routine
 345  *
 346  *  ixgbe_probe determines if the driver should be loaded on
 347  *  adapter based on PCI vendor/device id of the adapter.
 348  *
 349  *  return BUS_PROBE_DEFAULT on success, positive on failure
 350  *********************************************************************/
 351
 352 static int
 353 ixgbe_probe(device_t dev)
 354 {
 355         ixgbe_vendor_info_t *ent;
 356
 357         u16     pci_vendor_id = 0;
 358         u16     pci_device_id = 0;
 359         u16     pci_subvendor_id = 0;
 360         u16     pci_subdevice_id = 0;
 361         char    adapter_name[256];
 362
 363         INIT_DEBUGOUT("ixgbe_probe: begin");
 364
 365         pci_vendor_id = pci_get_vendor(dev);
 366         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 367                 return (ENXIO);
 368
 369         pci_device_id = pci_get_device(dev);
 370         pci_subvendor_id = pci_get_subvendor(dev);
 371         pci_subdevice_id = pci_get_subdevice(dev);
 372
 373         ent = ixgbe_vendor_info_array;
 374         while (ent->vendor_id != 0) {
 375                 if ((pci_vendor_id == ent->vendor_id) &&
 376                     (pci_device_id == ent->device_id) &&
 377
 378                     ((pci_subvendor_id == ent->subvendor_id) ||
 379                      (ent->subvendor_id == 0)) &&
 380
 381                     ((pci_subdevice_id == ent->subdevice_id) ||
 382                      (ent->subdevice_id == 0))) {
 383                         sprintf(adapter_name, "%s, Version - %s",
 384                                 ixgbe_strings[ent->index],
 385                                 ixgbe_driver_version);
 386                         device_set_desc_copy(dev, adapter_name);
 387                         ++ixgbe_total_ports;
 388                         return (BUS_PROBE_DEFAULT);
 389                 }
 390                 ent++;
 391         }
 392         return (ENXIO);
 393 }
 394
 395 /*********************************************************************
 396  *  Device initialization routine
 397  *
 398  *  The attach entry point is called when the driver is being loaded.
 399  *  This routine identifies the type of hardware, allocates all resources
 400  *  and initializes the hardware.
 401  *
 402  *  return 0 on success, positive on failure
 403  *********************************************************************/
 404
 405 static int
 406 ixgbe_attach(device_t dev)
 407 {
 408         struct adapter *adapter;
 409         struct ixgbe_hw *hw;
 410         int             error = 0;
 411         u16             csum;
 412         u32             ctrl_ext;
 413
 414         INIT_DEBUGOUT("ixgbe_attach: begin");
 415
 416         /* Allocate, clear, and link in our adapter structure */
 417         adapter = device_get_softc(dev);
 418         adapter->dev = adapter->osdep.dev = dev;
 419         hw = &adapter->hw;
 420
 421         /* Core Lock Init*/
 422         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 423
 424         /* SYSCTL APIs */
 425
 426         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 427                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 428                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 429                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 430
 431         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 432                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 433                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 434                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 435
 436         /*
 437         ** Allow a kind of speed control by forcing the autoneg
 438         ** advertised speed list to only a certain value, this
 439         ** supports 1G on 82599 devices, and 100Mb on x540.
 440         */
 441         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 442                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 443                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 444                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 445
 446         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 447                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 448                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 449                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 450
 451         /* Set up the timer callout */
 452         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 453
 454         /* Determine hardware revision */
 455         ixgbe_identify_hardware(adapter);
 456
 457         /* Do base PCI setup - map BAR0 */
 458         if (ixgbe_allocate_pci_resources(adapter)) {
 459                 device_printf(dev, "Allocation of PCI resources failed\n");
 460                 error = ENXIO;
 461                 goto err_out;
 462         }
 463
 464         /* Do descriptor calc and sanity checks */
 465         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 466             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 467                 device_printf(dev, "TXD config issue, using default!\n");
 468                 adapter->num_tx_desc = DEFAULT_TXD;
 469         } else
 470                 adapter->num_tx_desc = ixgbe_txd;
 471
 472         /*
 473         ** With many RX rings it is easy to exceed the
 474         ** system mbuf allocation. Tuning nmbclusters
 475         ** can alleviate this.
 476         */
 477         if (nmbclusters > 0 ) {
 478                 int s;
 479                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 480                 if (s > nmbclusters) {
 481                         device_printf(dev, "RX Descriptors exceed "
 482                             "system mbuf max, using default instead!\n");
 483                         ixgbe_rxd = DEFAULT_RXD;
 484                 }
 485         }
 486
 487         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 488             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 489                 device_printf(dev, "RXD config issue, using default!\n");
 490                 adapter->num_rx_desc = DEFAULT_RXD;
 491         } else
 492                 adapter->num_rx_desc = ixgbe_rxd;
 493
 494         /* Allocate our TX/RX Queues */
 495         if (ixgbe_allocate_queues(adapter)) {
 496                 error = ENOMEM;
 497                 goto err_out;
 498         }
 499
 500         /* Allocate multicast array memory. */
 501         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 502             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 503         if (adapter->mta == NULL) {
 504                 device_printf(dev, "Can not allocate multicast setup array\n");
 505                 error = ENOMEM;
 506                 goto err_late;
 507         }
 508
 509         /* Initialize the shared code */
 510         error = ixgbe_init_shared_code(hw);
 511         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 512                 /*
 513                 ** No optics in this port, set up
 514                 ** so the timer routine will probe
 515                 ** for later insertion.
 516                 */
 517                 adapter->sfp_probe = TRUE;
 518                 error = 0;
 519         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 520                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 521                 error = EIO;
 522                 goto err_late;
 523         } else if (error) {
 524                 device_printf(dev,"Unable to initialize the shared code\n");
 525                 error = EIO;
 526                 goto err_late;
 527         }
 528
 529         /* Make sure we have a good EEPROM before we read from it */
 530         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 531                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 532                 error = EIO;
 533                 goto err_late;
 534         }
 535
 536         error = ixgbe_init_hw(hw);
 537         switch (error) {
 538         case IXGBE_ERR_EEPROM_VERSION:
 539                 device_printf(dev, "This device is a pre-production adapter/"
 540                     "LOM.  Please be aware there may be issues associated "
 541                     "with your hardware.\n If you are experiencing problems "
 542                     "please contact your Intel or hardware representative "
 543                     "who provided you with this hardware.\n");
 544                 break;
 545         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 546                 device_printf(dev,"Unsupported SFP+ Module\n");
 547                 error = EIO;
 548                 goto err_late;
 549         case IXGBE_ERR_SFP_NOT_PRESENT:
 550                 device_printf(dev,"No SFP+ Module found\n");
 551                 /* falls thru */
 552         default:
 553                 break;
 554         }
 555
 556         /* Detect and set physical type */
 557         ixgbe_setup_optics(adapter);
 558
 559         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 560                 error = ixgbe_allocate_msix(adapter);
 561         else
 562                 error = ixgbe_allocate_legacy(adapter);
 563         if (error)
 564                 goto err_late;
 565
 566         /* Setup OS specific network interface */
 567         if (ixgbe_setup_interface(dev, adapter) != 0)
 568                 goto err_late;
 569
 570         /* Initialize statistics */
 571         ixgbe_update_stats_counters(adapter);
 572
 573         /* Register for VLAN events */
 574         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 575             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 576         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 577             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 578
 579         /* Print PCIE bus type/speed/width info */
 580         ixgbe_get_bus_info(hw);
 581         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 582             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 583             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 584             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 585             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 586             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 587             ("Unknown"));
 588
 589         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 590             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 591                 device_printf(dev, "PCI-Express bandwidth available"
 592                     " for this card\n     is not sufficient for"
 593                     " optimal performance.\n");
 594                 device_printf(dev, "For optimal performance a x8 "
 595                     "PCIE, or x4 PCIE 2 slot is required.\n");
 596         }
 597
 598         /* Set an initial default flow control value */
 599         adapter->fc =  ixgbe_fc_full;
 600
 601         /* let hardware know driver is loaded */
 602         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 603         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 604         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 605
 606         ixgbe_add_hw_stats(adapter);
 607
 608 #ifdef DEV_NETMAP
 609         ixgbe_netmap_attach(adapter);
 610 #endif /* DEV_NETMAP */
 611         INIT_DEBUGOUT("ixgbe_attach: end");
 612         return (0);
 613 err_late:
 614         ixgbe_free_transmit_structures(adapter);
 615         ixgbe_free_receive_structures(adapter);
 616 err_out:
 617         if (adapter->ifp != NULL)
 618                 if_free(adapter->ifp);
 619         ixgbe_free_pci_resources(adapter);
 620         free(adapter->mta, M_DEVBUF);
 621         return (error);
 622
 623 }
 624
 625 /*********************************************************************
 626  *  Device removal routine
 627  *
 628  *  The detach entry point is called when the driver is being removed.
 629  *  This routine stops the adapter and deallocates all the resources
 630  *  that were allocated for driver operation.
 631  *
 632  *  return 0 on success, positive on failure
 633  *********************************************************************/
 634
 635 static int
 636 ixgbe_detach(device_t dev)
 637 {
 638         struct adapter *adapter = device_get_softc(dev);
 639         struct ix_queue *que = adapter->queues;
 640         struct tx_ring *txr = adapter->tx_rings;
 641         u32     ctrl_ext;
 642
 643         INIT_DEBUGOUT("ixgbe_detach: begin");
 644
 645         /* Make sure VLANS are not using driver */
 646         if (adapter->ifp->if_vlantrunk != NULL) {
 647                 device_printf(dev,"Vlan in use, detach first\n");
 648                 return (EBUSY);
 649         }
 650
 651         IXGBE_CORE_LOCK(adapter);
 652         ixgbe_stop(adapter);
 653         IXGBE_CORE_UNLOCK(adapter);
 654
 655         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 656                 if (que->tq) {
 657 #ifndef IXGBE_LEGACY_TX
 658                         taskqueue_drain(que->tq, &txr->txq_task);
 659 #endif
 660                         taskqueue_drain(que->tq, &que->que_task);
 661                         taskqueue_free(que->tq);
 662                 }
 663         }
 664
 665         /* Drain the Link queue */
 666         if (adapter->tq) {
 667                 taskqueue_drain(adapter->tq, &adapter->link_task);
 668                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 669                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 670 #ifdef IXGBE_FDIR
 671                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 672 #endif
 673                 taskqueue_free(adapter->tq);
 674         }
 675
 676         /* let hardware know driver is unloading */
 677         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 678         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 679         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 680
 681         /* Unregister VLAN events */
 682         if (adapter->vlan_attach != NULL)
 683                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 684         if (adapter->vlan_detach != NULL)
 685                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 686
 687         ether_ifdetach(adapter->ifp);
 688         callout_drain(&adapter->timer);
 689 #ifdef DEV_NETMAP
 690         netmap_detach(adapter->ifp);
 691 #endif /* DEV_NETMAP */
 692         ixgbe_free_pci_resources(adapter);
 693         bus_generic_detach(dev);
 694         if_free(adapter->ifp);
 695
 696         ixgbe_free_transmit_structures(adapter);
 697         ixgbe_free_receive_structures(adapter);
 698         free(adapter->mta, M_DEVBUF);
 699
 700         IXGBE_CORE_LOCK_DESTROY(adapter);
 701         return (0);
 702 }
 703
 704 /*********************************************************************
 705  *
 706  *  Shutdown entry point
 707  *
 708  **********************************************************************/
 709
 710 static int
 711 ixgbe_shutdown(device_t dev)
 712 {
 713         struct adapter *adapter = device_get_softc(dev);
 714         IXGBE_CORE_LOCK(adapter);
 715         ixgbe_stop(adapter);
 716         IXGBE_CORE_UNLOCK(adapter);
 717         return (0);
 718 }
 719
 720
 721 #ifdef IXGBE_LEGACY_TX
 722 /*********************************************************************
 723  *  Transmit entry point
 724  *
 725  *  ixgbe_start is called by the stack to initiate a transmit.
 726  *  The driver will remain in this routine as long as there are
 727  *  packets to transmit and transmit resources are available.
 728  *  In case resources are not available stack is notified and
 729  *  the packet is requeued.
 730  **********************************************************************/
 731
 732 static void
 733 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 734 {
 735         struct mbuf    *m_head;
 736         struct adapter *adapter = txr->adapter;
 737
 738         IXGBE_TX_LOCK_ASSERT(txr);
 739
 740         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 741                 return;
 742         if (!adapter->link_active)
 743                 return;
 744
 745         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 746                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 747                         break;
 748
 749                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 750                 if (m_head == NULL)
 751                         break;
 752
 753                 if (ixgbe_xmit(txr, &m_head)) {
 754                         if (m_head != NULL)
 755                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 756                         break;
 757                 }
 758                 /* Send a copy of the frame to the BPF listener */
 759                 ETHER_BPF_MTAP(ifp, m_head);
 760
 761                 /* Set watchdog on */
 762                 txr->watchdog_time = ticks;
 763                 txr->queue_status = IXGBE_QUEUE_WORKING;
 764
 765         }
 766         return;
 767 }
 768
 769 /*
 770  * Legacy TX start - called by the stack, this
 771  * always uses the first tx ring, and should
 772  * not be used with multiqueue tx enabled.
 773  */
 774 static void
 775 ixgbe_start(struct ifnet *ifp)
 776 {
 777         struct adapter *adapter = ifp->if_softc;
 778         struct tx_ring  *txr = adapter->tx_rings;
 779
 780         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 781                 IXGBE_TX_LOCK(txr);
 782                 ixgbe_start_locked(txr, ifp);
 783                 IXGBE_TX_UNLOCK(txr);
 784         }
 785         return;
 786 }
 787
 788 #else /* ! IXGBE_LEGACY_TX */
 789
 790 /*
 791 ** Multiqueue Transmit driver
 792 **
 793 */
 794 static int
 795 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 796 {
 797         struct adapter  *adapter = ifp->if_softc;
 798         struct ix_queue *que;
 799         struct tx_ring  *txr;
 800         int             i = 0, err = 0;
 801
 802         /* Which queue to use */
 803         if ((m->m_flags & M_FLOWID) != 0)
 804                 i = m->m_pkthdr.flowid % adapter->num_queues;
 805         else
 806                 i = curcpu % adapter->num_queues;
 807
 808         txr = &adapter->tx_rings[i];
 809         que = &adapter->queues[i];
 810
 811         if (IXGBE_TX_TRYLOCK(txr)) {
 812                 err = ixgbe_mq_start_locked(ifp, txr, m);
 813                 IXGBE_TX_UNLOCK(txr);
 814         } else {
 815                 err = drbr_enqueue(ifp, txr->br, m);
 816                 taskqueue_enqueue(que->tq, &txr->txq_task);
 817         }
 818
 819         return (err);
 820 }
 821
 822 static int
 823 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 824 {
 825         struct adapter  *adapter = txr->adapter;
 826         struct mbuf     *next;
 827         int             enqueued, err = 0;
 828
 829         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 830             adapter->link_active == 0) {
 831                 if (m != NULL)
 832                         err = drbr_enqueue(ifp, txr->br, m);
 833                 return (err);
 834         }
 835
 836         enqueued = 0;
 837         if (m != NULL) {
 838                 err = drbr_enqueue(ifp, txr->br, m);
 839                 if (err) {
 840                         return (err);
 841                 }
 842         }
 843
 844         /* Process the queue */
 845         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 846                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 847                         if (next == NULL) {
 848                                 drbr_advance(ifp, txr->br);
 849                         } else {
 850                                 drbr_putback(ifp, txr->br, next);
 851                         }
 852                         break;
 853                 }
 854                 drbr_advance(ifp, txr->br);
 855                 enqueued++;
 856                 /* Send a copy of the frame to the BPF listener */
 857                 ETHER_BPF_MTAP(ifp, next);
 858                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 859                         break;
 860                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 861                         ixgbe_txeof(txr);
 862         }
 863
 864         if (enqueued > 0) {
 865                 /* Set watchdog on */
 866                 txr->queue_status = IXGBE_QUEUE_WORKING;
 867                 txr->watchdog_time = ticks;
 868         }
 869
 870         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 871                 ixgbe_txeof(txr);
 872
 873         return (err);
 874 }
 875
 876 /*
 877  * Called from a taskqueue to drain queued transmit packets.
 878  */
 879 static void
 880 ixgbe_deferred_mq_start(void *arg, int pending)
 881 {
 882         struct tx_ring *txr = arg;
 883         struct adapter *adapter = txr->adapter;
 884         struct ifnet *ifp = adapter->ifp;
 885
 886         IXGBE_TX_LOCK(txr);
 887         if (!drbr_empty(ifp, txr->br))
 888                 ixgbe_mq_start_locked(ifp, txr, NULL);
 889         IXGBE_TX_UNLOCK(txr);
 890 }
 891
 892 /*
 893 ** Flush all ring buffers
 894 */
 895 static void
 896 ixgbe_qflush(struct ifnet *ifp)
 897 {
 898         struct adapter  *adapter = ifp->if_softc;
 899         struct tx_ring  *txr = adapter->tx_rings;
 900         struct mbuf     *m;
 901
 902         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 903                 IXGBE_TX_LOCK(txr);
 904                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 905                         m_freem(m);
 906                 IXGBE_TX_UNLOCK(txr);
 907         }
 908         if_qflush(ifp);
 909 }
 910 #endif /* IXGBE_LEGACY_TX */
 911
 912 /*********************************************************************
 913  *  Ioctl entry point
 914  *
 915  *  ixgbe_ioctl is called when the user wants to configure the
 916  *  interface.
 917  *
 918  *  return 0 on success, positive on failure
 919  **********************************************************************/
 920
 921 static int
 922 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 923 {
 924         struct adapter  *adapter = ifp->if_softc;
 925         struct ixgbe_hw *hw = &adapter->hw;
 926         struct ifreq    *ifr = (struct ifreq *) data;
 927 #if defined(INET) || defined(INET6)
 928         struct ifaddr *ifa = (struct ifaddr *)data;
 929         bool            avoid_reset = FALSE;
 930 #endif
 931         int             error = 0;
 932
 933         switch (command) {
 934
 935         case SIOCSIFADDR:
 936 #ifdef INET
 937                 if (ifa->ifa_addr->sa_family == AF_INET)
 938                         avoid_reset = TRUE;
 939 #endif
 940 #ifdef INET6
 941                 if (ifa->ifa_addr->sa_family == AF_INET6)
 942                         avoid_reset = TRUE;
 943 #endif
 944 #if defined(INET) || defined(INET6)
 945                 /*
 946                 ** Calling init results in link renegotiation,
 947                 ** so we avoid doing it when possible.
 948                 */
 949                 if (avoid_reset) {
 950                         ifp->if_flags |= IFF_UP;
 951                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 952                                 ixgbe_init(adapter);
 953                         if (!(ifp->if_flags & IFF_NOARP))
 954                                 arp_ifinit(ifp, ifa);
 955                 } else
 956                         error = ether_ioctl(ifp, command, data);
 957 #endif
 958                 break;
 959         case SIOCSIFMTU:
 960                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 961                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 962                         error = EINVAL;
 963                 } else {
 964                         IXGBE_CORE_LOCK(adapter);
 965                         ifp->if_mtu = ifr->ifr_mtu;
 966                         adapter->max_frame_size =
 967                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 968                         ixgbe_init_locked(adapter);
 969                         IXGBE_CORE_UNLOCK(adapter);
 970                 }
 971                 break;
 972         case SIOCSIFFLAGS:
 973                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 974                 IXGBE_CORE_LOCK(adapter);
 975                 if (ifp->if_flags & IFF_UP) {
 976                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 977                                 if ((ifp->if_flags ^ adapter->if_flags) &
 978                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 979                                         ixgbe_set_promisc(adapter);
 980                                 }
 981                         } else
 982                                 ixgbe_init_locked(adapter);
 983                 } else
 984                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 985                                 ixgbe_stop(adapter);
 986                 adapter->if_flags = ifp->if_flags;
 987                 IXGBE_CORE_UNLOCK(adapter);
 988                 break;
 989         case SIOCADDMULTI:
 990         case SIOCDELMULTI:
 991                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 992                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 993                         IXGBE_CORE_LOCK(adapter);
 994                         ixgbe_disable_intr(adapter);
 995                         ixgbe_set_multi(adapter);
 996                         ixgbe_enable_intr(adapter);
 997                         IXGBE_CORE_UNLOCK(adapter);
 998                 }
 999                 break;
1000         case SIOCSIFMEDIA:
1001         case SIOCGIFMEDIA:
1002                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1003                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1004                 break;
1005         case SIOCSIFCAP:
1006         {
1007                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1008                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1009                 if (mask & IFCAP_HWCSUM)
1010                         ifp->if_capenable ^= IFCAP_HWCSUM;
1011                 if (mask & IFCAP_TSO4)
1012                         ifp->if_capenable ^= IFCAP_TSO4;
1013                 if (mask & IFCAP_TSO6)
1014                         ifp->if_capenable ^= IFCAP_TSO6;
1015                 if (mask & IFCAP_LRO)
1016                         ifp->if_capenable ^= IFCAP_LRO;
1017                 if (mask & IFCAP_VLAN_HWTAGGING)
1018                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1019                 if (mask & IFCAP_VLAN_HWFILTER)
1020                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1021                 if (mask & IFCAP_VLAN_HWTSO)
1022                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1023                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1024                         IXGBE_CORE_LOCK(adapter);
1025                         ixgbe_init_locked(adapter);
1026                         IXGBE_CORE_UNLOCK(adapter);
1027                 }
1028                 VLAN_CAPABILITIES(ifp);
1029                 break;
1030         }
1031         case SIOCGI2C:
1032         {
1033                 struct ixgbe_i2c_req    i2c;
1034                 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1035                 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1036                 if (error)
1037                         break;
1038                 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1039                         error = EINVAL;
1040                         break;
1041                 }
1042                 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1043                     i2c.dev_addr, i2c.data);
1044                 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1045                 break;
1046         }
1047         default:
1048                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1049                 error = ether_ioctl(ifp, command, data);
1050                 break;
1051         }
1052
1053         return (error);
1054 }
1055
1056 /*********************************************************************
1057  *  Init entry point
1058  *
1059  *  This routine is used in two ways. It is used by the stack as
1060  *  init entry point in network interface structure. It is also used
1061  *  by the driver as a hw/sw initialization routine to get to a
1062  *  consistent state.
1063  *
1064  *  return 0 on success, positive on failure
1065  **********************************************************************/
1066 #define IXGBE_MHADD_MFS_SHIFT 16
1067
1068 static void
1069 ixgbe_init_locked(struct adapter *adapter)
1070 {
1071         struct ifnet   *ifp = adapter->ifp;
1072         device_t        dev = adapter->dev;
1073         struct ixgbe_hw *hw = &adapter->hw;
1074         u32             k, txdctl, mhadd, gpie;
1075         u32             rxdctl, rxctrl;
1076
1077         mtx_assert(&adapter->core_mtx, MA_OWNED);
1078         INIT_DEBUGOUT("ixgbe_init: begin");
1079         hw->adapter_stopped = FALSE;
1080         ixgbe_stop_adapter(hw);
1081         callout_stop(&adapter->timer);
1082
1083         /* reprogram the RAR[0] in case user changed it. */
1084         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1085
1086         /* Get the latest mac address, User can use a LAA */
1087         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1088               IXGBE_ETH_LENGTH_OF_ADDRESS);
1089         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1090         hw->addr_ctrl.rar_used_count = 1;
1091
1092         /* Set the various hardware offload abilities */
1093         ifp->if_hwassist = 0;
1094         if (ifp->if_capenable & IFCAP_TSO)
1095                 ifp->if_hwassist |= CSUM_TSO;
1096         if (ifp->if_capenable & IFCAP_TXCSUM) {
1097                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1098 #if __FreeBSD_version >= 800000
1099                 if (hw->mac.type != ixgbe_mac_82598EB)
1100                         ifp->if_hwassist |= CSUM_SCTP;
1101 #endif
1102         }
1103
1104         /* Prepare transmit descriptors and buffers */
1105         if (ixgbe_setup_transmit_structures(adapter)) {
1106                 device_printf(dev,"Could not setup transmit structures\n");
1107                 ixgbe_stop(adapter);
1108                 return;
1109         }
1110
1111         ixgbe_init_hw(hw);
1112         ixgbe_initialize_transmit_units(adapter);
1113
1114         /* Setup Multicast table */
1115         ixgbe_set_multi(adapter);
1116
1117         /*
1118         ** Determine the correct mbuf pool
1119         ** for doing jumbo frames
1120         */
1121         if (adapter->max_frame_size <= 2048)
1122                 adapter->rx_mbuf_sz = MCLBYTES;
1123         else if (adapter->max_frame_size <= 4096)
1124                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1125         else if (adapter->max_frame_size <= 9216)
1126                 adapter->rx_mbuf_sz = MJUM9BYTES;
1127         else
1128                 adapter->rx_mbuf_sz = MJUM16BYTES;
1129
1130         /* Prepare receive descriptors and buffers */
1131         if (ixgbe_setup_receive_structures(adapter)) {
1132                 device_printf(dev,"Could not setup receive structures\n");
1133                 ixgbe_stop(adapter);
1134                 return;
1135         }
1136
1137         /* Configure RX settings */
1138         ixgbe_initialize_receive_units(adapter);
1139
1140         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1141
1142         /* Enable Fan Failure Interrupt */
1143         gpie |= IXGBE_SDP1_GPIEN;
1144
1145         /* Add for Module detection */
1146         if (hw->mac.type == ixgbe_mac_82599EB)
1147                 gpie |= IXGBE_SDP2_GPIEN;
1148
1149         /* Thermal Failure Detection */
1150         if (hw->mac.type == ixgbe_mac_X540)
1151                 gpie |= IXGBE_SDP0_GPIEN;
1152
1153         if (adapter->msix > 1) {
1154                 /* Enable Enhanced MSIX mode */
1155                 gpie |= IXGBE_GPIE_MSIX_MODE;
1156                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1157                     IXGBE_GPIE_OCD;
1158         }
1159         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1160
1161         /* Set MTU size */
1162         if (ifp->if_mtu > ETHERMTU) {
1163                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1164                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1165                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1166                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1167         }
1168
1169         /* Now enable all the queues */
1170
1171         for (int i = 0; i < adapter->num_queues; i++) {
1172                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1173                 txdctl |= IXGBE_TXDCTL_ENABLE;
1174                 /* Set WTHRESH to 8, burst writeback */
1175                 txdctl |= (8 << 16);
1176                 /*
1177                  * When the internal queue falls below PTHRESH (32),
1178                  * start prefetching as long as there are at least
1179                  * HTHRESH (1) buffers ready. The values are taken
1180                  * from the Intel linux driver 3.8.21.
1181                  * Prefetching enables tx line rate even with 1 queue.
1182                  */
1183                 txdctl |= (32 << 0) | (1 << 8);
1184                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1185         }
1186
1187         for (int i = 0; i < adapter->num_queues; i++) {
1188                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1189                 if (hw->mac.type == ixgbe_mac_82598EB) {
1190                         /*
1191                         ** PTHRESH = 21
1192                         ** HTHRESH = 4
1193                         ** WTHRESH = 8
1194                         */
1195                         rxdctl &= ~0x3FFFFF;
1196                         rxdctl |= 0x080420;
1197                 }
1198                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1199                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1200                 for (k = 0; k < 10; k++) {
1201                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1202                             IXGBE_RXDCTL_ENABLE)
1203                                 break;
1204                         else
1205                                 msec_delay(1);
1206                 }
1207                 wmb();
1208 #ifdef DEV_NETMAP
1209                 /*
1210                  * In netmap mode, we must preserve the buffers made
1211                  * available to userspace before the if_init()
1212                  * (this is true by default on the TX side, because
1213                  * init makes all buffers available to userspace).
1214                  *
1215                  * netmap_reset() and the device specific routines
1216                  * (e.g. ixgbe_setup_receive_rings()) map these
1217                  * buffers at the end of the NIC ring, so here we
1218                  * must set the RDT (tail) register to make sure
1219                  * they are not overwritten.
1220                  *
1221                  * In this driver the NIC ring starts at RDH = 0,
1222                  * RDT points to the last slot available for reception (?),
1223                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1224                  */
1225                 if (ifp->if_capenable & IFCAP_NETMAP) {
1226                         struct netmap_adapter *na = NA(adapter->ifp);
1227                         struct netmap_kring *kring = &na->rx_rings[i];
1228                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1229
1230                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1231                 } else
1232 #endif /* DEV_NETMAP */
1233                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1234         }
1235
1236         /* Set up VLAN support and filter */
1237         ixgbe_setup_vlan_hw_support(adapter);
1238
1239         /* Enable Receive engine */
1240         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1241         if (hw->mac.type == ixgbe_mac_82598EB)
1242                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1243         rxctrl |= IXGBE_RXCTRL_RXEN;
1244         ixgbe_enable_rx_dma(hw, rxctrl);
1245
1246         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1247
1248         /* Set up MSI/X routing */
1249         if (ixgbe_enable_msix)  {
1250                 ixgbe_configure_ivars(adapter);
1251                 /* Set up auto-mask */
1252                 if (hw->mac.type == ixgbe_mac_82598EB)
1253                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1254                 else {
1255                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1256                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1257                 }
1258         } else {  /* Simple settings for Legacy/MSI */
1259                 ixgbe_set_ivar(adapter, 0, 0, 0);
1260                 ixgbe_set_ivar(adapter, 0, 0, 1);
1261                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1262         }
1263
1264 #ifdef IXGBE_FDIR
1265         /* Init Flow director */
1266         if (hw->mac.type != ixgbe_mac_82598EB) {
1267                 u32 hdrm = 32 << fdir_pballoc;
1268
1269                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1270                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1271         }
1272 #endif
1273
1274         /*
1275         ** Check on any SFP devices that
1276         ** need to be kick-started
1277         */
1278         if (hw->phy.type == ixgbe_phy_none) {
1279                 int err = hw->phy.ops.identify(hw);
1280                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1281                         device_printf(dev,
1282                             "Unsupported SFP+ module type was detected.\n");
1283                         return;
1284                 }
1285         }
1286
1287         /* Set moderation on the Link interrupt */
1288         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1289
1290         /* Config/Enable Link */
1291         ixgbe_config_link(adapter);
1292
1293         /* Hardware Packet Buffer & Flow Control setup */
1294         {
1295                 u32 rxpb, frame, size, tmp;
1296
1297                 frame = adapter->max_frame_size;
1298
1299                 /* Calculate High Water */
1300                 if (hw->mac.type == ixgbe_mac_X540)
1301                         tmp = IXGBE_DV_X540(frame, frame);
1302                 else
1303                         tmp = IXGBE_DV(frame, frame);
1304                 size = IXGBE_BT2KB(tmp);
1305                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1306                 hw->fc.high_water[0] = rxpb - size;
1307
1308                 /* Now calculate Low Water */
1309                 if (hw->mac.type == ixgbe_mac_X540)
1310                         tmp = IXGBE_LOW_DV_X540(frame);
1311                 else
1312                         tmp = IXGBE_LOW_DV(frame);
1313                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1314
1315                 hw->fc.requested_mode = adapter->fc;
1316                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1317                 hw->fc.send_xon = TRUE;
1318         }
1319         /* Initialize the FC settings */
1320         ixgbe_start_hw(hw);
1321
1322         /* And now turn on interrupts */
1323         ixgbe_enable_intr(adapter);
1324
1325         /* Now inform the stack we're ready */
1326         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1327
1328         return;
1329 }
1330
1331 static void
1332 ixgbe_init(void *arg)
1333 {
1334         struct adapter *adapter = arg;
1335
1336         IXGBE_CORE_LOCK(adapter);
1337         ixgbe_init_locked(adapter);
1338         IXGBE_CORE_UNLOCK(adapter);
1339         return;
1340 }
1341
1342
1343 /*
1344 **
1345 ** MSIX Interrupt Handlers and Tasklets
1346 **
1347 */
1348
1349 static inline void
1350 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1351 {
1352         struct ixgbe_hw *hw = &adapter->hw;
1353         u64     queue = (u64)(1 << vector);
1354         u32     mask;
1355
1356         if (hw->mac.type == ixgbe_mac_82598EB) {
1357                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1358                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1359         } else {
1360                 mask = (queue & 0xFFFFFFFF);
1361                 if (mask)
1362                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1363                 mask = (queue >> 32);
1364                 if (mask)
1365                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1366         }
1367 }
1368
1369 static inline void
1370 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1371 {
1372         struct ixgbe_hw *hw = &adapter->hw;
1373         u64     queue = (u64)(1 << vector);
1374         u32     mask;
1375
1376         if (hw->mac.type == ixgbe_mac_82598EB) {
1377                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1379         } else {
1380                 mask = (queue & 0xFFFFFFFF);
1381                 if (mask)
1382                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1383                 mask = (queue >> 32);
1384                 if (mask)
1385                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1386         }
1387 }
1388
1389 static inline void
1390 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1391 {
1392         u32 mask;
1393
1394         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1395                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1396                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1397         } else {
1398                 mask = (queues & 0xFFFFFFFF);
1399                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1400                 mask = (queues >> 32);
1401                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1402         }
1403 }
1404
1405
1406 static void
1407 ixgbe_handle_que(void *context, int pending)
1408 {
1409         struct ix_queue *que = context;
1410         struct adapter  *adapter = que->adapter;
1411         struct tx_ring  *txr = que->txr;
1412         struct ifnet    *ifp = adapter->ifp;
1413         bool            more;
1414
1415         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1416                 more = ixgbe_rxeof(que);
1417                 IXGBE_TX_LOCK(txr);
1418                 ixgbe_txeof(txr);
1419 #ifndef IXGBE_LEGACY_TX
1420                 if (!drbr_empty(ifp, txr->br))
1421                         ixgbe_mq_start_locked(ifp, txr, NULL);
1422 #else
1423                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1424                         ixgbe_start_locked(txr, ifp);
1425 #endif
1426                 IXGBE_TX_UNLOCK(txr);
1427                 if (more) {
1428                         taskqueue_enqueue(que->tq, &que->que_task);
1429                         return;
1430                 }
1431         }
1432
1433         /* Reenable this interrupt */
1434         ixgbe_enable_queue(adapter, que->msix);
1435         return;
1436 }
1437
1438
1439 /*********************************************************************
1440  *
1441  *  Legacy Interrupt Service routine
1442  *
1443  **********************************************************************/
1444
1445 static void
1446 ixgbe_legacy_irq(void *arg)
1447 {
1448         struct ix_queue *que = arg;
1449         struct adapter  *adapter = que->adapter;
1450         struct ixgbe_hw *hw = &adapter->hw;
1451         struct          tx_ring *txr = adapter->tx_rings;
1452         bool            more_tx, more_rx;
1453         u32             reg_eicr, loop = MAX_LOOP;
1454
1455
1456         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1457
1458         ++que->irqs;
1459         if (reg_eicr == 0) {
1460                 ixgbe_enable_intr(adapter);
1461                 return;
1462         }
1463
1464         more_rx = ixgbe_rxeof(que);
1465
1466         IXGBE_TX_LOCK(txr);
1467         do {
1468                 more_tx = ixgbe_txeof(txr);
1469         } while (loop-- && more_tx);
1470         IXGBE_TX_UNLOCK(txr);
1471
1472         if (more_rx || more_tx)
1473                 taskqueue_enqueue(que->tq, &que->que_task);
1474
1475         /* Check for fan failure */
1476         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1477             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1478                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1479                     "REPLACE IMMEDIATELY!!\n");
1480                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1481         }
1482
1483         /* Link status change */
1484         if (reg_eicr & IXGBE_EICR_LSC)
1485                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1486
1487         ixgbe_enable_intr(adapter);
1488         return;
1489 }
1490
1491
1492 /*********************************************************************
1493  *
1494  *  MSIX Queue Interrupt Service routine
1495  *
1496  **********************************************************************/
1497 void
1498 ixgbe_msix_que(void *arg)
1499 {
1500         struct ix_queue *que = arg;
1501         struct adapter  *adapter = que->adapter;
1502         struct tx_ring  *txr = que->txr;
1503         struct rx_ring  *rxr = que->rxr;
1504         bool            more_tx, more_rx;
1505         u32             newitr = 0;
1506
1507         ixgbe_disable_queue(adapter, que->msix);
1508         ++que->irqs;
1509
1510         more_rx = ixgbe_rxeof(que);
1511
1512         IXGBE_TX_LOCK(txr);
1513         more_tx = ixgbe_txeof(txr);
1514         /*
1515         ** Make certain that if the stack
1516         ** has anything queued the task gets
1517         ** scheduled to handle it.
1518         */
1519 #ifdef IXGBE_LEGACY_TX
1520         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1521 #else
1522         if (!drbr_empty(adapter->ifp, txr->br))
1523 #endif
1524                 more_tx = 1;
1525         IXGBE_TX_UNLOCK(txr);
1526
1527         /* Do AIM now? */
1528
1529         if (ixgbe_enable_aim == FALSE)
1530                 goto no_calc;
1531         /*
1532         ** Do Adaptive Interrupt Moderation:
1533         **  - Write out last calculated setting
1534         **  - Calculate based on average size over
1535         **    the last interval.
1536         */
1537         if (que->eitr_setting)
1538                 IXGBE_WRITE_REG(&adapter->hw,
1539                     IXGBE_EITR(que->msix), que->eitr_setting);
1540
1541         que->eitr_setting = 0;
1542
1543         /* Idle, do nothing */
1544         if ((txr->bytes == 0) && (rxr->bytes == 0))
1545                 goto no_calc;
1546
1547         if ((txr->bytes) && (txr->packets))
1548                 newitr = txr->bytes/txr->packets;
1549         if ((rxr->bytes) && (rxr->packets))
1550                 newitr = max(newitr,
1551                     (rxr->bytes / rxr->packets));
1552         newitr += 24; /* account for hardware frame, crc */
1553
1554         /* set an upper boundary */
1555         newitr = min(newitr, 3000);
1556
1557         /* Be nice to the mid range */
1558         if ((newitr > 300) && (newitr < 1200))
1559                 newitr = (newitr / 3);
1560         else
1561                 newitr = (newitr / 2);
1562
1563         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1564                 newitr |= newitr << 16;
1565         else
1566                 newitr |= IXGBE_EITR_CNT_WDIS;
1567
1568         /* save for next interrupt */
1569         que->eitr_setting = newitr;
1570
1571         /* Reset state */
1572         txr->bytes = 0;
1573         txr->packets = 0;
1574         rxr->bytes = 0;
1575         rxr->packets = 0;
1576
1577 no_calc:
1578         if (more_tx || more_rx)
1579                 taskqueue_enqueue(que->tq, &que->que_task);
1580         else /* Reenable this interrupt */
1581                 ixgbe_enable_queue(adapter, que->msix);
1582         return;
1583 }
1584
1585
1586 static void
1587 ixgbe_msix_link(void *arg)
1588 {
1589         struct adapter  *adapter = arg;
1590         struct ixgbe_hw *hw = &adapter->hw;
1591         u32             reg_eicr;
1592
1593         ++adapter->link_irq;
1594
1595         /* First get the cause */
1596         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1597         /* Clear interrupt with write */
1598         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1599
1600         /* Link status change */
1601         if (reg_eicr & IXGBE_EICR_LSC)
1602                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1603
1604         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1605 #ifdef IXGBE_FDIR
1606                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1607                         /* This is probably overkill :) */
1608                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1609                                 return;
1610                         /* Disable the interrupt */
1611                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1612                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1613                 } else
1614 #endif
1615                 if (reg_eicr & IXGBE_EICR_ECC) {
1616                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1617                             "Please Reboot!!\n");
1618                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1619                 } else
1620
1621                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1622                         /* Clear the interrupt */
1623                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1624                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1625                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1626                         /* Clear the interrupt */
1627                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1628                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1629                 }
1630         }
1631
1632         /* Check for fan failure */
1633         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1634             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1635                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1636                     "REPLACE IMMEDIATELY!!\n");
1637                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1638         }
1639
1640         /* Check for over temp condition */
1641         if ((hw->mac.type == ixgbe_mac_X540) &&
1642             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1643                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1644                     "PHY IS SHUT DOWN!!\n");
1645                 device_printf(adapter->dev, "System shutdown required\n");
1646                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1647         }
1648
1649         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1650         return;
1651 }
1652
1653 /*********************************************************************
1654  *
1655  *  Media Ioctl callback
1656  *
1657  *  This routine is called whenever the user queries the status of
1658  *  the interface using ifconfig.
1659  *
1660  **********************************************************************/
1661 static void
1662 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1663 {
1664         struct adapter *adapter = ifp->if_softc;
1665
1666         INIT_DEBUGOUT("ixgbe_media_status: begin");
1667         IXGBE_CORE_LOCK(adapter);
1668         ixgbe_update_link_status(adapter);
1669
1670         ifmr->ifm_status = IFM_AVALID;
1671         ifmr->ifm_active = IFM_ETHER;
1672
1673         if (!adapter->link_active) {
1674                 IXGBE_CORE_UNLOCK(adapter);
1675                 return;
1676         }
1677
1678         ifmr->ifm_status |= IFM_ACTIVE;
1679
1680         switch (adapter->link_speed) {
1681                 case IXGBE_LINK_SPEED_100_FULL:
1682                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1683                         break;
1684                 case IXGBE_LINK_SPEED_1GB_FULL:
1685                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1686                         break;
1687                 case IXGBE_LINK_SPEED_10GB_FULL:
1688                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1689                         break;
1690         }
1691
1692         IXGBE_CORE_UNLOCK(adapter);
1693
1694         return;
1695 }
1696
1697 /*********************************************************************
1698  *
1699  *  Media Ioctl callback
1700  *
1701  *  This routine is called when the user changes speed/duplex using
1702  *  media/mediopt option with ifconfig.
1703  *
1704  **********************************************************************/
1705 static int
1706 ixgbe_media_change(struct ifnet * ifp)
1707 {
1708         struct adapter *adapter = ifp->if_softc;
1709         struct ifmedia *ifm = &adapter->media;
1710
1711         INIT_DEBUGOUT("ixgbe_media_change: begin");
1712
1713         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1714                 return (EINVAL);
1715
1716         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1717         case IFM_AUTO:
1718                 adapter->hw.phy.autoneg_advertised =
1719                     IXGBE_LINK_SPEED_100_FULL |
1720                     IXGBE_LINK_SPEED_1GB_FULL |
1721                     IXGBE_LINK_SPEED_10GB_FULL;
1722                 break;
1723         default:
1724                 device_printf(adapter->dev, "Only auto media type\n");
1725                 return (EINVAL);
1726         }
1727
1728         return (0);
1729 }
1730
1731 /*********************************************************************
1732  *
1733  *  This routine maps the mbufs to tx descriptors, allowing the
1734  *  TX engine to transmit the packets.
1735  *      - return 0 on success, positive on failure
1736  *
1737  **********************************************************************/
1738
1739 static int
1740 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1741 {
1742         struct adapter  *adapter = txr->adapter;
1743         u32             olinfo_status = 0, cmd_type_len;
1744         int             i, j, error, nsegs;
1745         int             first;
1746         bool            remap = TRUE;
1747         struct mbuf     *m_head;
1748         bus_dma_segment_t segs[adapter->num_segs];
1749         bus_dmamap_t    map;
1750         struct ixgbe_tx_buf *txbuf;
1751         union ixgbe_adv_tx_desc *txd = NULL;
1752
1753         m_head = *m_headp;
1754
1755         /* Basic descriptor defines */
1756         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1757             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1758
1759         if (m_head->m_flags & M_VLANTAG)
1760                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1761
1762         /*
1763          * Important to capture the first descriptor
1764          * used because it will contain the index of
1765          * the one we tell the hardware to report back
1766          */
1767         first = txr->next_avail_desc;
1768         txbuf = &txr->tx_buffers[first];
1769         map = txbuf->map;
1770
1771         /*
1772          * Map the packet for DMA.
1773          */
1774 retry:
1775         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1776             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1777
1778         if (__predict_false(error)) {
1779                 struct mbuf *m;
1780
1781                 switch (error) {
1782                 case EFBIG:
1783                         /* Try it again? - one try */
1784                         if (remap == TRUE) {
1785                                 remap = FALSE;
1786                                 m = m_defrag(*m_headp, M_NOWAIT);
1787                                 if (m == NULL) {
1788                                         adapter->mbuf_defrag_failed++;
1789                                         m_freem(*m_headp);
1790                                         *m_headp = NULL;
1791                                         return (ENOBUFS);
1792                                 }
1793                                 *m_headp = m;
1794                                 goto retry;
1795                         } else
1796                                 return (error);
1797                 case ENOMEM:
1798                         txr->no_tx_dma_setup++;
1799                         return (error);
1800                 default:
1801                         txr->no_tx_dma_setup++;
1802                         m_freem(*m_headp);
1803                         *m_headp = NULL;
1804                         return (error);
1805                 }
1806         }
1807
1808         /* Make certain there are enough descriptors */
1809         if (nsegs > txr->tx_avail - 2) {
1810                 txr->no_desc_avail++;
1811                 bus_dmamap_unload(txr->txtag, map);
1812                 return (ENOBUFS);
1813         }
1814         m_head = *m_headp;
1815
1816         /*
1817         ** Set up the appropriate offload context
1818         ** this will consume the first descriptor
1819         */
1820         error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1821         if (__predict_false(error)) {
1822                 if (error == ENOBUFS)
1823                         *m_headp = NULL;
1824                 return (error);
1825         }
1826
1827 #ifdef IXGBE_FDIR
1828         /* Do the flow director magic */
1829         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1830                 ++txr->atr_count;
1831                 if (txr->atr_count >= atr_sample_rate) {
1832                         ixgbe_atr(txr, m_head);
1833                         txr->atr_count = 0;
1834                 }
1835         }
1836 #endif
1837
1838         i = txr->next_avail_desc;
1839         for (j = 0; j < nsegs; j++) {
1840                 bus_size_t seglen;
1841                 bus_addr_t segaddr;
1842
1843                 txbuf = &txr->tx_buffers[i];
1844                 txd = &txr->tx_base[i];
1845                 seglen = segs[j].ds_len;
1846                 segaddr = htole64(segs[j].ds_addr);
1847
1848                 txd->read.buffer_addr = segaddr;
1849                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1850                     cmd_type_len |seglen);
1851                 txd->read.olinfo_status = htole32(olinfo_status);
1852
1853                 if (++i == txr->num_desc)
1854                         i = 0;
1855         }
1856
1857         txd->read.cmd_type_len |=
1858             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1859         txr->tx_avail -= nsegs;
1860         txr->next_avail_desc = i;
1861
1862         txbuf->m_head = m_head;
1863         /*
1864         ** Here we swap the map so the last descriptor,
1865         ** which gets the completion interrupt has the
1866         ** real map, and the first descriptor gets the
1867         ** unused map from this descriptor.
1868         */
1869         txr->tx_buffers[first].map = txbuf->map;
1870         txbuf->map = map;
1871         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1872
1873         /* Set the EOP descriptor that will be marked done */
1874         txbuf = &txr->tx_buffers[first];
1875         txbuf->eop = txd;
1876
1877         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1878             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1879         /*
1880          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1881          * hardware that this frame is available to transmit.
1882          */
1883         ++txr->total_packets;
1884         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1885
1886         return (0);
1887
1888 }
1889
1890 static void
1891 ixgbe_set_promisc(struct adapter *adapter)
1892 {
1893         u_int32_t       reg_rctl;
1894         struct ifnet   *ifp = adapter->ifp;
1895
1896         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1897         reg_rctl &= (~IXGBE_FCTRL_UPE);
1898         reg_rctl &= (~IXGBE_FCTRL_MPE);
1899         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1900
1901         if (ifp->if_flags & IFF_PROMISC) {
1902                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1903                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1904         } else if (ifp->if_flags & IFF_ALLMULTI) {
1905                 reg_rctl |= IXGBE_FCTRL_MPE;
1906                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1907                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1908         }
1909         return;
1910 }
1911
1912
1913 /*********************************************************************
1914  *  Multicast Update
1915  *
1916  *  This routine is called whenever multicast address list is updated.
1917  *
1918  **********************************************************************/
1919 #define IXGBE_RAR_ENTRIES 16
1920
1921 static void
1922 ixgbe_set_multi(struct adapter *adapter)
1923 {
1924         u32     fctrl;
1925         u8      *mta;
1926         u8      *update_ptr;
1927         struct  ifmultiaddr *ifma;
1928         int     mcnt = 0;
1929         struct ifnet   *ifp = adapter->ifp;
1930
1931         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1932
1933         mta = adapter->mta;
1934         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1935             MAX_NUM_MULTICAST_ADDRESSES);
1936
1937 #if __FreeBSD_version < 800000
1938         IF_ADDR_LOCK(ifp);
1939 #else
1940         if_maddr_rlock(ifp);
1941 #endif
1942         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1943                 if (ifma->ifma_addr->sa_family != AF_LINK)
1944                         continue;
1945                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1946                         break;
1947                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1948                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1949                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1950                 mcnt++;
1951         }
1952 #if __FreeBSD_version < 800000
1953         IF_ADDR_UNLOCK(ifp);
1954 #else
1955         if_maddr_runlock(ifp);
1956 #endif
1957
1958         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1959         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1960         if (ifp->if_flags & IFF_PROMISC)
1961                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1962         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1963             ifp->if_flags & IFF_ALLMULTI) {
1964                 fctrl |= IXGBE_FCTRL_MPE;
1965                 fctrl &= ~IXGBE_FCTRL_UPE;
1966         } else
1967                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1968
1969         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1970
1971         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
1972                 update_ptr = mta;
1973                 ixgbe_update_mc_addr_list(&adapter->hw,
1974                     update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1975         }
1976
1977         return;
1978 }
1979
1980 /*
1981  * This is an iterator function now needed by the multicast
1982  * shared code. It simply feeds the shared code routine the
1983  * addresses in the array of ixgbe_set_multi() one by one.
1984  */
1985 static u8 *
1986 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1987 {
1988         u8 *addr = *update_ptr;
1989         u8 *newptr;
1990         *vmdq = 0;
1991
1992         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1993         *update_ptr = newptr;
1994         return addr;
1995 }
1996
1997
1998 /*********************************************************************
1999  *  Timer routine
2000  *
2001  *  This routine checks for link status,updates statistics,
2002  *  and runs the watchdog check.
2003  *
2004  **********************************************************************/
2005
2006 static void
2007 ixgbe_local_timer(void *arg)
2008 {
2009         struct adapter  *adapter = arg;
2010         device_t        dev = adapter->dev;
2011         struct ix_queue *que = adapter->queues;
2012         struct tx_ring  *txr = adapter->tx_rings;
2013         int             hung = 0, paused = 0;
2014
2015         mtx_assert(&adapter->core_mtx, MA_OWNED);
2016
2017         /* Check for pluggable optics */
2018         if (adapter->sfp_probe)
2019                 if (!ixgbe_sfp_probe(adapter))
2020                         goto out; /* Nothing to do */
2021
2022         ixgbe_update_link_status(adapter);
2023         ixgbe_update_stats_counters(adapter);
2024
2025         /*
2026          * If the interface has been paused
2027          * then don't do the watchdog check
2028          */
2029         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2030                 paused = 1;
2031
2032         /*
2033         ** Check the TX queues status
2034         **      - watchdog only if all queues show hung
2035         */
2036         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2037                 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2038                     (paused == 0))
2039                         ++hung;
2040                 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2041                         taskqueue_enqueue(que->tq, &que->que_task);
2042         }
2043         /* Only truely watchdog if all queues show hung */
2044         if (hung == adapter->num_queues)
2045                 goto watchdog;
2046
2047 out:
2048         ixgbe_rearm_queues(adapter, adapter->que_mask);
2049         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2050         return;
2051
2052 watchdog:
2053         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2054         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2055             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2056             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2057         device_printf(dev,"TX(%d) desc avail = %d,"
2058             "Next TX to Clean = %d\n",
2059             txr->me, txr->tx_avail, txr->next_to_clean);
2060         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2061         adapter->watchdog_events++;
2062         ixgbe_init_locked(adapter);
2063 }
2064
2065 /*
2066 ** Note: this routine updates the OS on the link state
2067 **      the real check of the hardware only happens with
2068 **      a link interrupt.
2069 */
2070 static void
2071 ixgbe_update_link_status(struct adapter *adapter)
2072 {
2073         struct ifnet    *ifp = adapter->ifp;
2074         device_t dev = adapter->dev;
2075
2076
2077         if (adapter->link_up){
2078                 if (adapter->link_active == FALSE) {
2079                         if (bootverbose)
2080                                 device_printf(dev,"Link is up %d Gbps %s \n",
2081                                     ((adapter->link_speed == 128)? 10:1),
2082                                     "Full Duplex");
2083                         adapter->link_active = TRUE;
2084                         /* Update any Flow Control changes */
2085                         ixgbe_fc_enable(&adapter->hw);
2086                         if_link_state_change(ifp, LINK_STATE_UP);
2087                 }
2088         } else { /* Link down */
2089                 if (adapter->link_active == TRUE) {
2090                         if (bootverbose)
2091                                 device_printf(dev,"Link is Down\n");
2092                         if_link_state_change(ifp, LINK_STATE_DOWN);
2093                         adapter->link_active = FALSE;
2094                 }
2095         }
2096
2097         return;
2098 }
2099
2100
2101 /*********************************************************************
2102  *
2103  *  This routine disables all traffic on the adapter by issuing a
2104  *  global reset on the MAC and deallocates TX/RX buffers.
2105  *
2106  **********************************************************************/
2107
2108 static void
2109 ixgbe_stop(void *arg)
2110 {
2111         struct ifnet   *ifp;
2112         struct adapter *adapter = arg;
2113         struct ixgbe_hw *hw = &adapter->hw;
2114         ifp = adapter->ifp;
2115
2116         mtx_assert(&adapter->core_mtx, MA_OWNED);
2117
2118         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2119         ixgbe_disable_intr(adapter);
2120         callout_stop(&adapter->timer);
2121
2122         /* Let the stack know...*/
2123         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2124
2125         ixgbe_reset_hw(hw);
2126         hw->adapter_stopped = FALSE;
2127         ixgbe_stop_adapter(hw);
2128         /* Turn off the laser */
2129         if (hw->phy.multispeed_fiber)
2130                 ixgbe_disable_tx_laser(hw);
2131
2132         /* reprogram the RAR[0] in case user changed it. */
2133         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2134
2135         return;
2136 }
2137
2138
2139 /*********************************************************************
2140  *
2141  *  Determine hardware revision.
2142  *
2143  **********************************************************************/
2144 static void
2145 ixgbe_identify_hardware(struct adapter *adapter)
2146 {
2147         device_t        dev = adapter->dev;
2148         struct ixgbe_hw *hw = &adapter->hw;
2149
2150         /* Save off the information about this board */
2151         hw->vendor_id = pci_get_vendor(dev);
2152         hw->device_id = pci_get_device(dev);
2153         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2154         hw->subsystem_vendor_id =
2155             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2156         hw->subsystem_device_id =
2157             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2158
2159         /* We need this here to set the num_segs below */
2160         ixgbe_set_mac_type(hw);
2161
2162         /* Pick up the 82599 and VF settings */
2163         if (hw->mac.type != ixgbe_mac_82598EB) {
2164                 hw->phy.smart_speed = ixgbe_smart_speed;
2165                 adapter->num_segs = IXGBE_82599_SCATTER;
2166         } else
2167                 adapter->num_segs = IXGBE_82598_SCATTER;
2168
2169         return;
2170 }
2171
2172 /*********************************************************************
2173  *
2174  *  Determine optic type
2175  *
2176  **********************************************************************/
2177 static void
2178 ixgbe_setup_optics(struct adapter *adapter)
2179 {
2180         struct ixgbe_hw *hw = &adapter->hw;
2181         int             layer;
2182
2183         layer = ixgbe_get_supported_physical_layer(hw);
2184
2185         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2186                 adapter->optics = IFM_10G_T;
2187                 return;
2188         }
2189
2190         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2191                 adapter->optics = IFM_1000_T;
2192                 return;
2193         }
2194
2195         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2196                 adapter->optics = IFM_1000_SX;
2197                 return;
2198         }
2199
2200         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2201             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2202                 adapter->optics = IFM_10G_LR;
2203                 return;
2204         }
2205
2206         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2207                 adapter->optics = IFM_10G_SR;
2208                 return;
2209         }
2210
2211         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2212                 adapter->optics = IFM_10G_TWINAX;
2213                 return;
2214         }
2215
2216         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2217             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2218                 adapter->optics = IFM_10G_CX4;
2219                 return;
2220         }
2221
2222         /* If we get here just set the default */
2223         adapter->optics = IFM_ETHER | IFM_AUTO;
2224         return;
2225 }
2226
2227 /*********************************************************************
2228  *
2229  *  Setup the Legacy or MSI Interrupt handler
2230  *
2231  **********************************************************************/
2232 static int
2233 ixgbe_allocate_legacy(struct adapter *adapter)
2234 {
2235         device_t        dev = adapter->dev;
2236         struct          ix_queue *que = adapter->queues;
2237 #ifndef IXGBE_LEGACY_TX
2238         struct tx_ring          *txr = adapter->tx_rings;
2239 #endif
2240         int             error, rid = 0;
2241
2242         /* MSI RID at 1 */
2243         if (adapter->msix == 1)
2244                 rid = 1;
2245
2246         /* We allocate a single interrupt resource */
2247         adapter->res = bus_alloc_resource_any(dev,
2248             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2249         if (adapter->res == NULL) {
2250                 device_printf(dev, "Unable to allocate bus resource: "
2251                     "interrupt\n");
2252                 return (ENXIO);
2253         }
2254
2255         /*
2256          * Try allocating a fast interrupt and the associated deferred
2257          * processing contexts.
2258          */
2259 #ifndef IXGBE_LEGACY_TX
2260         TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2261 #endif
2262         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2263         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2264             taskqueue_thread_enqueue, &que->tq);
2265         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2266             device_get_nameunit(adapter->dev));
2267
2268         /* Tasklets for Link, SFP and Multispeed Fiber */
2269         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2270         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2271         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2272 #ifdef IXGBE_FDIR
2273         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2274 #endif
2275         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2276             taskqueue_thread_enqueue, &adapter->tq);
2277         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2278             device_get_nameunit(adapter->dev));
2279
2280         if ((error = bus_setup_intr(dev, adapter->res,
2281             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2282             que, &adapter->tag)) != 0) {
2283                 device_printf(dev, "Failed to register fast interrupt "
2284                     "handler: %d\n", error);
2285                 taskqueue_free(que->tq);
2286                 taskqueue_free(adapter->tq);
2287                 que->tq = NULL;
2288                 adapter->tq = NULL;
2289                 return (error);
2290         }
2291         /* For simplicity in the handlers */
2292         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2293
2294         return (0);
2295 }
2296
2297
2298 /*********************************************************************
2299  *
2300  *  Setup MSIX Interrupt resources and handlers
2301  *
2302  **********************************************************************/
2303 static int
2304 ixgbe_allocate_msix(struct adapter *adapter)
2305 {
2306         device_t        dev = adapter->dev;
2307         struct          ix_queue *que = adapter->queues;
2308         struct          tx_ring *txr = adapter->tx_rings;
2309         int             error, rid, vector = 0;
2310
2311         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2312                 rid = vector + 1;
2313                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2314                     RF_SHAREABLE | RF_ACTIVE);
2315                 if (que->res == NULL) {
2316                         device_printf(dev,"Unable to allocate"
2317                             " bus resource: que interrupt [%d]\n", vector);
2318                         return (ENXIO);
2319                 }
2320                 /* Set the handler function */
2321                 error = bus_setup_intr(dev, que->res,
2322                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2323                     ixgbe_msix_que, que, &que->tag);
2324                 if (error) {
2325                         que->res = NULL;
2326                         device_printf(dev, "Failed to register QUE handler");
2327                         return (error);
2328                 }
2329 #if __FreeBSD_version >= 800504
2330                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2331 #endif
2332                 que->msix = vector;
2333                 adapter->que_mask |= (u64)(1 << que->msix);
2334                 /*
2335                 ** Bind the msix vector, and thus the
2336                 ** ring to the corresponding cpu.
2337                 */
2338                 if (adapter->num_queues > 1)
2339                         bus_bind_intr(dev, que->res, i);
2340
2341 #ifndef IXGBE_LEGACY_TX
2342                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2343 #endif
2344                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2345                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2346                     taskqueue_thread_enqueue, &que->tq);
2347                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2348                     device_get_nameunit(adapter->dev));
2349         }
2350
2351         /* and Link */
2352         rid = vector + 1;
2353         adapter->res = bus_alloc_resource_any(dev,
2354             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2355         if (!adapter->res) {
2356                 device_printf(dev,"Unable to allocate"
2357             " bus resource: Link interrupt [%d]\n", rid);
2358                 return (ENXIO);
2359         }
2360         /* Set the link handler function */
2361         error = bus_setup_intr(dev, adapter->res,
2362             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2363             ixgbe_msix_link, adapter, &adapter->tag);
2364         if (error) {
2365                 adapter->res = NULL;
2366                 device_printf(dev, "Failed to register LINK handler");
2367                 return (error);
2368         }
2369 #if __FreeBSD_version >= 800504
2370         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2371 #endif
2372         adapter->linkvec = vector;
2373         /* Tasklets for Link, SFP and Multispeed Fiber */
2374         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2375         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2376         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2377 #ifdef IXGBE_FDIR
2378         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2379 #endif
2380         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2381             taskqueue_thread_enqueue, &adapter->tq);
2382         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2383             device_get_nameunit(adapter->dev));
2384
2385         return (0);
2386 }
2387
2388 /*
2389  * Setup Either MSI/X or MSI
2390  */
2391 static int
2392 ixgbe_setup_msix(struct adapter *adapter)
2393 {
2394         device_t dev = adapter->dev;
2395         int rid, want, queues, msgs;
2396
2397         /* Override by tuneable */
2398         if (ixgbe_enable_msix == 0)
2399                 goto msi;
2400
2401         /* First try MSI/X */
2402         rid = PCIR_BAR(MSIX_82598_BAR);
2403         adapter->msix_mem = bus_alloc_resource_any(dev,
2404             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2405         if (!adapter->msix_mem) {
2406                 rid += 4;       /* 82599 maps in higher BAR */
2407                 adapter->msix_mem = bus_alloc_resource_any(dev,
2408                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2409         }
2410         if (!adapter->msix_mem) {
2411                 /* May not be enabled */
2412                 device_printf(adapter->dev,
2413                     "Unable to map MSIX table \n");
2414                 goto msi;
2415         }
2416
2417         msgs = pci_msix_count(dev);
2418         if (msgs == 0) { /* system has msix disabled */
2419                 bus_release_resource(dev, SYS_RES_MEMORY,
2420                     rid, adapter->msix_mem);
2421                 adapter->msix_mem = NULL;
2422                 goto msi;
2423         }
2424
2425         /* Figure out a reasonable auto config value */
2426         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2427
2428         if (ixgbe_num_queues != 0)
2429                 queues = ixgbe_num_queues;
2430         /* Set max queues to 8 when autoconfiguring */
2431         else if ((ixgbe_num_queues == 0) && (queues > 8))
2432                 queues = 8;
2433
2434         /*
2435         ** Want one vector (RX/TX pair) per queue
2436         ** plus an additional for Link.
2437         */
2438         want = queues + 1;
2439         if (msgs >= want)
2440                 msgs = want;
2441         else {
2442                 device_printf(adapter->dev,
2443                     "MSIX Configuration Problem, "
2444                     "%d vectors but %d queues wanted!\n",
2445                     msgs, want);
2446                 return (0); /* Will go to Legacy setup */
2447         }
2448         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2449                 device_printf(adapter->dev,
2450                     "Using MSIX interrupts with %d vectors\n", msgs);
2451                 adapter->num_queues = queues;
2452                 return (msgs);
2453         }
2454 msi:
2455         msgs = pci_msi_count(dev);
2456         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2457                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2458         else
2459                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2460         return (msgs);
2461 }
2462
2463
2464 static int
2465 ixgbe_allocate_pci_resources(struct adapter *adapter)
2466 {
2467         int             rid;
2468         device_t        dev = adapter->dev;
2469
2470         rid = PCIR_BAR(0);
2471         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2472             &rid, RF_ACTIVE);
2473
2474         if (!(adapter->pci_mem)) {
2475                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2476                 return (ENXIO);
2477         }
2478
2479         adapter->osdep.mem_bus_space_tag =
2480                 rman_get_bustag(adapter->pci_mem);
2481         adapter->osdep.mem_bus_space_handle =
2482                 rman_get_bushandle(adapter->pci_mem);
2483         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2484
2485         /* Legacy defaults */
2486         adapter->num_queues = 1;
2487         adapter->hw.back = &adapter->osdep;
2488
2489         /*
2490         ** Now setup MSI or MSI/X, should
2491         ** return us the number of supported
2492         ** vectors. (Will be 1 for MSI)
2493         */
2494         adapter->msix = ixgbe_setup_msix(adapter);
2495         return (0);
2496 }
2497
2498 static void
2499 ixgbe_free_pci_resources(struct adapter * adapter)
2500 {
2501         struct          ix_queue *que = adapter->queues;
2502         device_t        dev = adapter->dev;
2503         int             rid, memrid;
2504
2505         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2506                 memrid = PCIR_BAR(MSIX_82598_BAR);
2507         else
2508                 memrid = PCIR_BAR(MSIX_82599_BAR);
2509
2510         /*
2511         ** There is a slight possibility of a failure mode
2512         ** in attach that will result in entering this function
2513         ** before interrupt resources have been initialized, and
2514         ** in that case we do not want to execute the loops below
2515         ** We can detect this reliably by the state of the adapter
2516         ** res pointer.
2517         */
2518         if (adapter->res == NULL)
2519                 goto mem;
2520
2521         /*
2522         **  Release all msix queue resources:
2523         */
2524         for (int i = 0; i < adapter->num_queues; i++, que++) {
2525                 rid = que->msix + 1;
2526                 if (que->tag != NULL) {
2527                         bus_teardown_intr(dev, que->res, que->tag);
2528                         que->tag = NULL;
2529                 }
2530                 if (que->res != NULL)
2531                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2532         }
2533
2534
2535         /* Clean the Legacy or Link interrupt last */
2536         if (adapter->linkvec) /* we are doing MSIX */
2537                 rid = adapter->linkvec + 1;
2538         else
2539                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2540
2541         if (adapter->tag != NULL) {
2542                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2543                 adapter->tag = NULL;
2544         }
2545         if (adapter->res != NULL)
2546                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2547
2548 mem:
2549         if (adapter->msix)
2550                 pci_release_msi(dev);
2551
2552         if (adapter->msix_mem != NULL)
2553                 bus_release_resource(dev, SYS_RES_MEMORY,
2554                     memrid, adapter->msix_mem);
2555
2556         if (adapter->pci_mem != NULL)
2557                 bus_release_resource(dev, SYS_RES_MEMORY,
2558                     PCIR_BAR(0), adapter->pci_mem);
2559
2560         return;
2561 }
2562
2563 /*********************************************************************
2564  *
2565  *  Setup networking device structure and register an interface.
2566  *
2567  **********************************************************************/
2568 static int
2569 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2570 {
2571         struct ixgbe_hw *hw = &adapter->hw;
2572         struct ifnet   *ifp;
2573
2574         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2575
2576         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2577         if (ifp == NULL) {
2578                 device_printf(dev, "can not allocate ifnet structure\n");
2579                 return (-1);
2580         }
2581         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2582         ifp->if_baudrate = 1000000000;
2583         ifp->if_init = ixgbe_init;
2584         ifp->if_softc = adapter;
2585         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2586         ifp->if_ioctl = ixgbe_ioctl;
2587 #ifndef IXGBE_LEGACY_TX
2588         ifp->if_transmit = ixgbe_mq_start;
2589         ifp->if_qflush = ixgbe_qflush;
2590 #else
2591         ifp->if_start = ixgbe_start;
2592         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2593 #endif
2594
2595         ether_ifattach(ifp, adapter->hw.mac.addr);
2596
2597         adapter->max_frame_size =
2598             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2599
2600         /*
2601          * Tell the upper layer(s) we support long frames.
2602          */
2603         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2604
2605         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2606         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2607         ifp->if_capabilities |= IFCAP_LRO;
2608         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2609                              |  IFCAP_VLAN_HWTSO
2610                              |  IFCAP_VLAN_MTU;
2611         ifp->if_capenable = ifp->if_capabilities;
2612
2613         /*
2614         ** Don't turn this on by default, if vlans are
2615         ** created on another pseudo device (eg. lagg)
2616         ** then vlan events are not passed thru, breaking
2617         ** operation, but with HW FILTER off it works. If
2618         ** using vlans directly on the ixgbe driver you can
2619         ** enable this and get full hardware tag filtering.
2620         */
2621         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2622
2623         /*
2624          * Specify the media types supported by this adapter and register
2625          * callbacks to update media and link information
2626          */
2627         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2628                      ixgbe_media_status);
2629         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2630         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2631         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2632                 ifmedia_add(&adapter->media,
2633                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2634                 ifmedia_add(&adapter->media,
2635                     IFM_ETHER | IFM_1000_T, 0, NULL);
2636         }
2637         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2638         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2639
2640         return (0);
2641 }
2642
2643 static void
2644 ixgbe_config_link(struct adapter *adapter)
2645 {
2646         struct ixgbe_hw *hw = &adapter->hw;
2647         u32     autoneg, err = 0;
2648         bool    sfp, negotiate;
2649
2650         sfp = ixgbe_is_sfp(hw);
2651
2652         if (sfp) {
2653                 if (hw->phy.multispeed_fiber) {
2654                         hw->mac.ops.setup_sfp(hw);
2655                         ixgbe_enable_tx_laser(hw);
2656                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2657                 } else
2658                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2659         } else {
2660                 if (hw->mac.ops.check_link)
2661                         err = ixgbe_check_link(hw, &adapter->link_speed,
2662                             &adapter->link_up, FALSE);
2663                 if (err)
2664                         goto out;
2665                 autoneg = hw->phy.autoneg_advertised;
2666                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2667                         err  = hw->mac.ops.get_link_capabilities(hw,
2668                             &autoneg, &negotiate);
2669                 if (err)
2670                         goto out;
2671                 if (hw->mac.ops.setup_link)
2672                         err = hw->mac.ops.setup_link(hw,
2673                             autoneg, adapter->link_up);
2674         }
2675 out:
2676         return;
2677 }
2678
2679 /********************************************************************
2680  * Manage DMA'able memory.
2681  *******************************************************************/
2682 static void
2683 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2684 {
2685         if (error)
2686                 return;
2687         *(bus_addr_t *) arg = segs->ds_addr;
2688         return;
2689 }
2690
2691 static int
2692 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2693                 struct ixgbe_dma_alloc *dma, int mapflags)
2694 {
2695         device_t dev = adapter->dev;
2696         int             r;
2697
2698         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2699                                DBA_ALIGN, 0,    /* alignment, bounds */
2700                                BUS_SPACE_MAXADDR,       /* lowaddr */
2701                                BUS_SPACE_MAXADDR,       /* highaddr */
2702                                NULL, NULL,      /* filter, filterarg */
2703                                size,    /* maxsize */
2704                                1,       /* nsegments */
2705                                size,    /* maxsegsize */
2706                                BUS_DMA_ALLOCNOW,        /* flags */
2707                                NULL,    /* lockfunc */
2708                                NULL,    /* lockfuncarg */
2709                                &dma->dma_tag);
2710         if (r != 0) {
2711                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2712                        "error %u\n", r);
2713                 goto fail_0;
2714         }
2715         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2716                              BUS_DMA_NOWAIT, &dma->dma_map);
2717         if (r != 0) {
2718                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2719                        "error %u\n", r);
2720                 goto fail_1;
2721         }
2722         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2723                             size,
2724                             ixgbe_dmamap_cb,
2725                             &dma->dma_paddr,
2726                             mapflags | BUS_DMA_NOWAIT);
2727         if (r != 0) {
2728                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2729                        "error %u\n", r);
2730                 goto fail_2;
2731         }
2732         dma->dma_size = size;
2733         return (0);
2734 fail_2:
2735         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2736 fail_1:
2737         bus_dma_tag_destroy(dma->dma_tag);
2738 fail_0:
2739         dma->dma_map = NULL;
2740         dma->dma_tag = NULL;
2741         return (r);
2742 }
2743
2744 static void
2745 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2746 {
2747         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2748             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2749         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2750         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2751         bus_dma_tag_destroy(dma->dma_tag);
2752 }
2753
2754
2755 /*********************************************************************
2756  *
2757  *  Allocate memory for the transmit and receive rings, and then
2758  *  the descriptors associated with each, called only once at attach.
2759  *
2760  **********************************************************************/
2761 static int
2762 ixgbe_allocate_queues(struct adapter *adapter)
2763 {
2764         device_t        dev = adapter->dev;
2765         struct ix_queue *que;
2766         struct tx_ring  *txr;
2767         struct rx_ring  *rxr;
2768         int rsize, tsize, error = IXGBE_SUCCESS;
2769         int txconf = 0, rxconf = 0;
2770
2771         /* First allocate the top level queue structs */
2772         if (!(adapter->queues =
2773             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2774             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2775                 device_printf(dev, "Unable to allocate queue memory\n");
2776                 error = ENOMEM;
2777                 goto fail;
2778         }
2779
2780         /* First allocate the TX ring struct memory */
2781         if (!(adapter->tx_rings =
2782             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2783             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2784                 device_printf(dev, "Unable to allocate TX ring memory\n");
2785                 error = ENOMEM;
2786                 goto tx_fail;
2787         }
2788
2789         /* Next allocate the RX */
2790         if (!(adapter->rx_rings =
2791             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2792             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2793                 device_printf(dev, "Unable to allocate RX ring memory\n");
2794                 error = ENOMEM;
2795                 goto rx_fail;
2796         }
2797
2798         /* For the ring itself */
2799         tsize = roundup2(adapter->num_tx_desc *
2800             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2801
2802         /*
2803          * Now set up the TX queues, txconf is needed to handle the
2804          * possibility that things fail midcourse and we need to
2805          * undo memory gracefully
2806          */
2807         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2808                 /* Set up some basics */
2809                 txr = &adapter->tx_rings[i];
2810                 txr->adapter = adapter;
2811                 txr->me = i;
2812                 txr->num_desc = adapter->num_tx_desc;
2813
2814                 /* Initialize the TX side lock */
2815                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2816                     device_get_nameunit(dev), txr->me);
2817                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2818
2819                 if (ixgbe_dma_malloc(adapter, tsize,
2820                         &txr->txdma, BUS_DMA_NOWAIT)) {
2821                         device_printf(dev,
2822                             "Unable to allocate TX Descriptor memory\n");
2823                         error = ENOMEM;
2824                         goto err_tx_desc;
2825                 }
2826                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2827                 bzero((void *)txr->tx_base, tsize);
2828
2829                 /* Now allocate transmit buffers for the ring */
2830                 if (ixgbe_allocate_transmit_buffers(txr)) {
2831                         device_printf(dev,
2832                             "Critical Failure setting up transmit buffers\n");
2833                         error = ENOMEM;
2834                         goto err_tx_desc;
2835                 }
2836 #ifndef IXGBE_LEGACY_TX
2837                 /* Allocate a buf ring */
2838                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2839                     M_WAITOK, &txr->tx_mtx);
2840                 if (txr->br == NULL) {
2841                         device_printf(dev,
2842                             "Critical Failure setting up buf ring\n");
2843                         error = ENOMEM;
2844                         goto err_tx_desc;
2845                 }
2846 #endif
2847         }
2848
2849         /*
2850          * Next the RX queues...
2851          */
2852         rsize = roundup2(adapter->num_rx_desc *
2853             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2854         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2855                 rxr = &adapter->rx_rings[i];
2856                 /* Set up some basics */
2857                 rxr->adapter = adapter;
2858                 rxr->me = i;
2859                 rxr->num_desc = adapter->num_rx_desc;
2860
2861                 /* Initialize the RX side lock */
2862                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2863                     device_get_nameunit(dev), rxr->me);
2864                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2865
2866                 if (ixgbe_dma_malloc(adapter, rsize,
2867                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2868                         device_printf(dev,
2869                             "Unable to allocate RxDescriptor memory\n");
2870                         error = ENOMEM;
2871                         goto err_rx_desc;
2872                 }
2873                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2874                 bzero((void *)rxr->rx_base, rsize);
2875
2876                 /* Allocate receive buffers for the ring*/
2877                 if (ixgbe_allocate_receive_buffers(rxr)) {
2878                         device_printf(dev,
2879                             "Critical Failure setting up receive buffers\n");
2880                         error = ENOMEM;
2881                         goto err_rx_desc;
2882                 }
2883         }
2884
2885         /*
2886         ** Finally set up the queue holding structs
2887         */
2888         for (int i = 0; i < adapter->num_queues; i++) {
2889                 que = &adapter->queues[i];
2890                 que->adapter = adapter;
2891                 que->txr = &adapter->tx_rings[i];
2892                 que->rxr = &adapter->rx_rings[i];
2893         }
2894
2895         return (0);
2896
2897 err_rx_desc:
2898         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2899                 ixgbe_dma_free(adapter, &rxr->rxdma);
2900 err_tx_desc:
2901         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2902                 ixgbe_dma_free(adapter, &txr->txdma);
2903         free(adapter->rx_rings, M_DEVBUF);
2904 rx_fail:
2905         free(adapter->tx_rings, M_DEVBUF);
2906 tx_fail:
2907         free(adapter->queues, M_DEVBUF);
2908 fail:
2909         return (error);
2910 }
2911
2912 /*********************************************************************
2913  *
2914  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2915  *  the information needed to transmit a packet on the wire. This is
2916  *  called only once at attach, setup is done every reset.
2917  *
2918  **********************************************************************/
2919 static int
2920 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2921 {
2922         struct adapter *adapter = txr->adapter;
2923         device_t dev = adapter->dev;
2924         struct ixgbe_tx_buf *txbuf;
2925         int error, i;
2926
2927         /*
2928          * Setup DMA descriptor areas.
2929          */
2930         if ((error = bus_dma_tag_create(
2931                                bus_get_dma_tag(adapter->dev),   /* parent */
2932                                1, 0,            /* alignment, bounds */
2933                                BUS_SPACE_MAXADDR,       /* lowaddr */
2934                                BUS_SPACE_MAXADDR,       /* highaddr */
2935                                NULL, NULL,              /* filter, filterarg */
2936                                IXGBE_TSO_SIZE,          /* maxsize */
2937                                adapter->num_segs,       /* nsegments */
2938                                PAGE_SIZE,               /* maxsegsize */
2939                                0,                       /* flags */
2940                                NULL,                    /* lockfunc */
2941                                NULL,                    /* lockfuncarg */
2942                                &txr->txtag))) {
2943                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2944                 goto fail;
2945         }
2946
2947         if (!(txr->tx_buffers =
2948             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2949             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2950                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2951                 error = ENOMEM;
2952                 goto fail;
2953         }
2954
2955         /* Create the descriptor buffer dma maps */
2956         txbuf = txr->tx_buffers;
2957         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2958                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2959                 if (error != 0) {
2960                         device_printf(dev, "Unable to create TX DMA map\n");
2961                         goto fail;
2962                 }
2963         }
2964
2965         return 0;
2966 fail:
2967         /* We free all, it handles case where we are in the middle */
2968         ixgbe_free_transmit_structures(adapter);
2969         return (error);
2970 }
2971
2972 /*********************************************************************
2973  *
2974  *  Initialize a transmit ring.
2975  *
2976  **********************************************************************/
2977 static void
2978 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2979 {
2980         struct adapter *adapter = txr->adapter;
2981         struct ixgbe_tx_buf *txbuf;
2982         int i;
2983 #ifdef DEV_NETMAP
2984         struct netmap_adapter *na = NA(adapter->ifp);
2985         struct netmap_slot *slot;
2986 #endif /* DEV_NETMAP */
2987
2988         /* Clear the old ring contents */
2989         IXGBE_TX_LOCK(txr);
2990 #ifdef DEV_NETMAP
2991         /*
2992          * (under lock): if in netmap mode, do some consistency
2993          * checks and set slot to entry 0 of the netmap ring.
2994          */
2995         slot = netmap_reset(na, NR_TX, txr->me, 0);
2996 #endif /* DEV_NETMAP */
2997         bzero((void *)txr->tx_base,
2998               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2999         /* Reset indices */
3000         txr->next_avail_desc = 0;
3001         txr->next_to_clean = 0;
3002
3003         /* Free any existing tx buffers. */
3004         txbuf = txr->tx_buffers;
3005         for (i = 0; i < txr->num_desc; i++, txbuf++) {
3006                 if (txbuf->m_head != NULL) {
3007                         bus_dmamap_sync(txr->txtag, txbuf->map,
3008                             BUS_DMASYNC_POSTWRITE);
3009                         bus_dmamap_unload(txr->txtag, txbuf->map);
3010                         m_freem(txbuf->m_head);
3011                         txbuf->m_head = NULL;
3012                 }
3013 #ifdef DEV_NETMAP
3014                 /*
3015                  * In netmap mode, set the map for the packet buffer.
3016                  * NOTE: Some drivers (not this one) also need to set
3017                  * the physical buffer address in the NIC ring.
3018                  * Slots in the netmap ring (indexed by "si") are
3019                  * kring->nkr_hwofs positions "ahead" wrt the
3020                  * corresponding slot in the NIC ring. In some drivers
3021                  * (not here) nkr_hwofs can be negative. Function
3022                  * netmap_idx_n2k() handles wraparounds properly.
3023                  */
3024                 if (slot) {
3025                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3026                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3027                 }
3028 #endif /* DEV_NETMAP */
3029                 /* Clear the EOP descriptor pointer */
3030                 txbuf->eop = NULL;
3031         }
3032
3033 #ifdef IXGBE_FDIR
3034         /* Set the rate at which we sample packets */
3035         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3036                 txr->atr_sample = atr_sample_rate;
3037 #endif
3038
3039         /* Set number of descriptors available */
3040         txr->tx_avail = adapter->num_tx_desc;
3041
3042         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3043             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3044         IXGBE_TX_UNLOCK(txr);
3045 }
3046
3047 /*********************************************************************
3048  *
3049  *  Initialize all transmit rings.
3050  *
3051  **********************************************************************/
3052 static int
3053 ixgbe_setup_transmit_structures(struct adapter *adapter)
3054 {
3055         struct tx_ring *txr = adapter->tx_rings;
3056
3057         for (int i = 0; i < adapter->num_queues; i++, txr++)
3058                 ixgbe_setup_transmit_ring(txr);
3059
3060         return (0);
3061 }
3062
3063 /*********************************************************************
3064  *
3065  *  Enable transmit unit.
3066  *
3067  **********************************************************************/
3068 static void
3069 ixgbe_initialize_transmit_units(struct adapter *adapter)
3070 {
3071         struct tx_ring  *txr = adapter->tx_rings;
3072         struct ixgbe_hw *hw = &adapter->hw;
3073
3074         /* Setup the Base and Length of the Tx Descriptor Ring */
3075
3076         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3077                 u64     tdba = txr->txdma.dma_paddr;
3078                 u32     txctrl;
3079
3080                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3081                        (tdba & 0x00000000ffffffffULL));
3082                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3083                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3084                     adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3085
3086                 /* Setup the HW Tx Head and Tail descriptor pointers */
3087                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3088                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3089
3090                 /* Setup Transmit Descriptor Cmd Settings */
3091                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3092                 txr->queue_status = IXGBE_QUEUE_IDLE;
3093
3094                 /* Set the processing limit */
3095                 txr->process_limit = ixgbe_tx_process_limit;
3096
3097                 /* Disable Head Writeback */
3098                 switch (hw->mac.type) {
3099                 case ixgbe_mac_82598EB:
3100                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3101                         break;
3102                 case ixgbe_mac_82599EB:
3103                 case ixgbe_mac_X540:
3104                 default:
3105                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3106                         break;
3107                 }
3108                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3109                 switch (hw->mac.type) {
3110                 case ixgbe_mac_82598EB:
3111                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3112                         break;
3113                 case ixgbe_mac_82599EB:
3114                 case ixgbe_mac_X540:
3115                 default:
3116                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3117                         break;
3118                 }
3119
3120         }
3121
3122         if (hw->mac.type != ixgbe_mac_82598EB) {
3123                 u32 dmatxctl, rttdcs;
3124                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3125                 dmatxctl |= IXGBE_DMATXCTL_TE;
3126                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3127                 /* Disable arbiter to set MTQC */
3128                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3129                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3130                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3131                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3132                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3133                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3134         }
3135
3136         return;
3137 }
3138
3139 /*********************************************************************
3140  *
3141  *  Free all transmit rings.
3142  *
3143  **********************************************************************/
3144 static void
3145 ixgbe_free_transmit_structures(struct adapter *adapter)
3146 {
3147         struct tx_ring *txr = adapter->tx_rings;
3148
3149         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3150                 IXGBE_TX_LOCK(txr);
3151                 ixgbe_free_transmit_buffers(txr);
3152                 ixgbe_dma_free(adapter, &txr->txdma);
3153                 IXGBE_TX_UNLOCK(txr);
3154                 IXGBE_TX_LOCK_DESTROY(txr);
3155         }
3156         free(adapter->tx_rings, M_DEVBUF);
3157 }
3158
3159 /*********************************************************************
3160  *
3161  *  Free transmit ring related data structures.
3162  *
3163  **********************************************************************/
3164 static void
3165 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3166 {
3167         struct adapter *adapter = txr->adapter;
3168         struct ixgbe_tx_buf *tx_buffer;
3169         int             i;
3170
3171         INIT_DEBUGOUT("free_transmit_ring: begin");
3172
3173         if (txr->tx_buffers == NULL)
3174                 return;
3175
3176         tx_buffer = txr->tx_buffers;
3177         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3178                 if (tx_buffer->m_head != NULL) {
3179                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3180                             BUS_DMASYNC_POSTWRITE);
3181                         bus_dmamap_unload(txr->txtag,
3182                             tx_buffer->map);
3183                         m_freem(tx_buffer->m_head);
3184                         tx_buffer->m_head = NULL;
3185                         if (tx_buffer->map != NULL) {
3186                                 bus_dmamap_destroy(txr->txtag,
3187                                     tx_buffer->map);
3188                                 tx_buffer->map = NULL;
3189                         }
3190                 } else if (tx_buffer->map != NULL) {
3191                         bus_dmamap_unload(txr->txtag,
3192                             tx_buffer->map);
3193                         bus_dmamap_destroy(txr->txtag,
3194                             tx_buffer->map);
3195                         tx_buffer->map = NULL;
3196                 }
3197         }
3198 #ifdef IXGBE_LEGACY_TX
3199         if (txr->br != NULL)
3200                 buf_ring_free(txr->br, M_DEVBUF);
3201 #endif
3202         if (txr->tx_buffers != NULL) {
3203                 free(txr->tx_buffers, M_DEVBUF);
3204                 txr->tx_buffers = NULL;
3205         }
3206         if (txr->txtag != NULL) {
3207                 bus_dma_tag_destroy(txr->txtag);
3208                 txr->txtag = NULL;
3209         }
3210         return;
3211 }
3212
3213 /*********************************************************************
3214  *
3215  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
3216  *
3217  **********************************************************************/
3218
3219 static int
3220 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3221     u32 *cmd_type_len, u32 *olinfo_status)
3222 {
3223         struct ixgbe_adv_tx_context_desc *TXD;
3224         struct ether_vlan_header *eh;
3225         struct ip *ip;
3226         struct ip6_hdr *ip6;
3227         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3228         int     ehdrlen, ip_hlen = 0;
3229         u16     etype;
3230         u8      ipproto = 0;
3231         int     offload = TRUE;
3232         int     ctxd = txr->next_avail_desc;
3233         u16     vtag = 0;
3234
3235         /* First check if TSO is to be used */
3236         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3237                 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3238
3239         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3240                 offload = FALSE;
3241
3242         /* Indicate the whole packet as payload when not doing TSO */
3243         *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3244
3245         /* Now ready a context descriptor */
3246         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3247
3248         /*
3249         ** In advanced descriptors the vlan tag must
3250         ** be placed into the context descriptor. Hence
3251         ** we need to make one even if not doing offloads.
3252         */
3253         if (mp->m_flags & M_VLANTAG) {
3254                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3255                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3256         } else if (offload == FALSE) /* ... no offload to do */
3257                 return (0);
3258
3259         /*
3260          * Determine where frame payload starts.
3261          * Jump over vlan headers if already present,
3262          * helpful for QinQ too.
3263          */
3264         eh = mtod(mp, struct ether_vlan_header *);
3265         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3266                 etype = ntohs(eh->evl_proto);
3267                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3268         } else {
3269                 etype = ntohs(eh->evl_encap_proto);
3270                 ehdrlen = ETHER_HDR_LEN;
3271         }
3272
3273         /* Set the ether header length */
3274         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3275
3276         switch (etype) {
3277                 case ETHERTYPE_IP:
3278                         ip = (struct ip *)(mp->m_data + ehdrlen);
3279                         ip_hlen = ip->ip_hl << 2;
3280                         ipproto = ip->ip_p;
3281                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3282                         break;
3283                 case ETHERTYPE_IPV6:
3284                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3285                         ip_hlen = sizeof(struct ip6_hdr);
3286                         /* XXX-BZ this will go badly in case of ext hdrs. */
3287                         ipproto = ip6->ip6_nxt;
3288                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3289                         break;
3290                 default:
3291                         offload = FALSE;
3292                         break;
3293         }
3294
3295         vlan_macip_lens |= ip_hlen;
3296         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3297
3298         switch (ipproto) {
3299                 case IPPROTO_TCP:
3300                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3301                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3302                         break;
3303
3304                 case IPPROTO_UDP:
3305                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3306                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3307                         break;
3308
3309 #if __FreeBSD_version >= 800000
3310                 case IPPROTO_SCTP:
3311                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3312                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3313                         break;
3314 #endif
3315                 default:
3316                         offload = FALSE;
3317                         break;
3318         }
3319
3320         if (offload) /* For the TX descriptor setup */
3321                 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3322
3323         /* Now copy bits into descriptor */
3324         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3325         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3326         TXD->seqnum_seed = htole32(0);
3327         TXD->mss_l4len_idx = htole32(0);
3328
3329         /* We've consumed the first desc, adjust counters */
3330         if (++ctxd == txr->num_desc)
3331                 ctxd = 0;
3332         txr->next_avail_desc = ctxd;
3333         --txr->tx_avail;
3334
3335         return (0);
3336 }
3337
3338 /**********************************************************************
3339  *
3340  *  Setup work for hardware segmentation offload (TSO) on
3341  *  adapters using advanced tx descriptors
3342  *
3343  **********************************************************************/
3344 static int
3345 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3346     u32 *cmd_type_len, u32 *olinfo_status)
3347 {
3348         struct ixgbe_adv_tx_context_desc *TXD;
3349         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3350         u32 mss_l4len_idx = 0, paylen;
3351         u16 vtag = 0, eh_type;
3352         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3353         struct ether_vlan_header *eh;
3354 #ifdef INET6
3355         struct ip6_hdr *ip6;
3356 #endif
3357 #ifdef INET
3358         struct ip *ip;
3359 #endif
3360         struct tcphdr *th;
3361
3362
3363         /*
3364          * Determine where frame payload starts.
3365          * Jump over vlan headers if already present
3366          */
3367         eh = mtod(mp, struct ether_vlan_header *);
3368         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3369                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3370                 eh_type = eh->evl_proto;
3371         } else {
3372                 ehdrlen = ETHER_HDR_LEN;
3373                 eh_type = eh->evl_encap_proto;
3374         }
3375
3376         switch (ntohs(eh_type)) {
3377 #ifdef INET6
3378         case ETHERTYPE_IPV6:
3379                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3380                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3381                 if (ip6->ip6_nxt != IPPROTO_TCP)
3382                         return (ENXIO);
3383                 ip_hlen = sizeof(struct ip6_hdr);
3384                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3385                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3386                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3387                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3388                 break;
3389 #endif
3390 #ifdef INET
3391         case ETHERTYPE_IP:
3392                 ip = (struct ip *)(mp->m_data + ehdrlen);
3393                 if (ip->ip_p != IPPROTO_TCP)
3394                         return (ENXIO);
3395                 ip->ip_sum = 0;
3396                 ip_hlen = ip->ip_hl << 2;
3397                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3398                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3399                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3400                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3401                 /* Tell transmit desc to also do IPv4 checksum. */
3402                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3403                 break;
3404 #endif
3405         default:
3406                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3407                     __func__, ntohs(eh_type));
3408                 break;
3409         }
3410
3411         ctxd = txr->next_avail_desc;
3412         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3413
3414         tcp_hlen = th->th_off << 2;
3415
3416         /* This is used in the transmit desc in encap */
3417         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3418
3419         /* VLAN MACLEN IPLEN */
3420         if (mp->m_flags & M_VLANTAG) {
3421                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3422                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3423         }
3424
3425         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3426         vlan_macip_lens |= ip_hlen;
3427         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3428
3429         /* ADV DTYPE TUCMD */
3430         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3431         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3432         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3433
3434         /* MSS L4LEN IDX */
3435         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3436         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3437         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3438
3439         TXD->seqnum_seed = htole32(0);
3440
3441         if (++ctxd == txr->num_desc)
3442                 ctxd = 0;
3443
3444         txr->tx_avail--;
3445         txr->next_avail_desc = ctxd;
3446         *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3447         *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3448         *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3449         ++txr->tso_tx;
3450         return (0);
3451 }
3452
3453 #ifdef IXGBE_FDIR
3454 /*
3455 ** This routine parses packet headers so that Flow
3456 ** Director can make a hashed filter table entry
3457 ** allowing traffic flows to be identified and kept
3458 ** on the same cpu.  This would be a performance
3459 ** hit, but we only do it at IXGBE_FDIR_RATE of
3460 ** packets.
3461 */
3462 static void
3463 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3464 {
3465         struct adapter                  *adapter = txr->adapter;
3466         struct ix_queue                 *que;
3467         struct ip                       *ip;
3468         struct tcphdr                   *th;
3469         struct udphdr                   *uh;
3470         struct ether_vlan_header        *eh;
3471         union ixgbe_atr_hash_dword      input = {.dword = 0};
3472         union ixgbe_atr_hash_dword      common = {.dword = 0};
3473         int                             ehdrlen, ip_hlen;
3474         u16                             etype;
3475
3476         eh = mtod(mp, struct ether_vlan_header *);
3477         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3478                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3479                 etype = eh->evl_proto;
3480         } else {
3481                 ehdrlen = ETHER_HDR_LEN;
3482                 etype = eh->evl_encap_proto;
3483         }
3484
3485         /* Only handling IPv4 */
3486         if (etype != htons(ETHERTYPE_IP))
3487                 return;
3488
3489         ip = (struct ip *)(mp->m_data + ehdrlen);
3490         ip_hlen = ip->ip_hl << 2;
3491
3492         /* check if we're UDP or TCP */
3493         switch (ip->ip_p) {
3494         case IPPROTO_TCP:
3495                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3496                 /* src and dst are inverted */
3497                 common.port.dst ^= th->th_sport;
3498                 common.port.src ^= th->th_dport;
3499                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3500                 break;
3501         case IPPROTO_UDP:
3502                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3503                 /* src and dst are inverted */
3504                 common.port.dst ^= uh->uh_sport;
3505                 common.port.src ^= uh->uh_dport;
3506                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3507                 break;
3508         default:
3509                 return;
3510         }
3511
3512         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3513         if (mp->m_pkthdr.ether_vtag)
3514                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3515         else
3516                 common.flex_bytes ^= etype;
3517         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3518
3519         que = &adapter->queues[txr->me];
3520         /*
3521         ** This assumes the Rx queue and Tx
3522         ** queue are bound to the same CPU
3523         */
3524         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3525             input, common, que->msix);
3526 }
3527 #endif /* IXGBE_FDIR */
3528
3529 /**********************************************************************
3530  *
3531  *  Examine each tx_buffer in the used queue. If the hardware is done
3532  *  processing the packet then free associated resources. The
3533  *  tx_buffer is put back on the free queue.
3534  *
3535  **********************************************************************/
3536 static bool
3537 ixgbe_txeof(struct tx_ring *txr)
3538 {
3539         struct adapter          *adapter = txr->adapter;
3540         struct ifnet            *ifp = adapter->ifp;
3541         u32                     work, processed = 0;
3542         u16                     limit = txr->process_limit;
3543         struct ixgbe_tx_buf     *buf;
3544         union ixgbe_adv_tx_desc *txd;
3545
3546         mtx_assert(&txr->tx_mtx, MA_OWNED);
3547
3548 #ifdef DEV_NETMAP
3549         if (ifp->if_capenable & IFCAP_NETMAP) {
3550                 struct netmap_adapter *na = NA(ifp);
3551                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3552                 txd = txr->tx_base;
3553                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3554                     BUS_DMASYNC_POSTREAD);
3555                 /*
3556                  * In netmap mode, all the work is done in the context
3557                  * of the client thread. Interrupt handlers only wake up
3558                  * clients, which may be sleeping on individual rings
3559                  * or on a global resource for all rings.
3560                  * To implement tx interrupt mitigation, we wake up the client
3561                  * thread roughly every half ring, even if the NIC interrupts
3562                  * more frequently. This is implemented as follows:
3563                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3564                  *   the slot that should wake up the thread (nkr_num_slots
3565                  *   means the user thread should not be woken up);
3566                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3567                  *   or the slot has the DD bit set.
3568                  *
3569                  * When the driver has separate locks, we need to
3570                  * release and re-acquire txlock to avoid deadlocks.
3571                  * XXX see if we can find a better way.
3572                  */
3573                 if (!netmap_mitigate ||
3574                     (kring->nr_kflags < kring->nkr_num_slots &&
3575                     txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3576                         kring->nr_kflags = kring->nkr_num_slots;
3577                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3578                         IXGBE_TX_UNLOCK(txr);
3579                         IXGBE_CORE_LOCK(adapter);
3580                         selwakeuppri(&na->tx_si, PI_NET);
3581                         IXGBE_CORE_UNLOCK(adapter);
3582                         IXGBE_TX_LOCK(txr);
3583                 }
3584                 return FALSE;
3585         }
3586 #endif /* DEV_NETMAP */
3587
3588         if (txr->tx_avail == txr->num_desc) {
3589                 txr->queue_status = IXGBE_QUEUE_IDLE;
3590                 return FALSE;
3591         }
3592
3593         /* Get work starting point */
3594         work = txr->next_to_clean;
3595         buf = &txr->tx_buffers[work];
3596         txd = &txr->tx_base[work];
3597         work -= txr->num_desc; /* The distance to ring end */
3598         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3599             BUS_DMASYNC_POSTREAD);
3600
3601         do {
3602                 union ixgbe_adv_tx_desc *eop= buf->eop;
3603                 if (eop == NULL) /* No work */
3604                         break;
3605
3606                 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3607                         break;  /* I/O not complete */
3608
3609                 if (buf->m_head) {
3610                         txr->bytes +=
3611                             buf->m_head->m_pkthdr.len;
3612                         bus_dmamap_sync(txr->txtag,
3613                             buf->map,
3614                             BUS_DMASYNC_POSTWRITE);
3615                         bus_dmamap_unload(txr->txtag,
3616                             buf->map);
3617                         m_freem(buf->m_head);
3618                         buf->m_head = NULL;
3619                         buf->map = NULL;
3620                 }
3621                 buf->eop = NULL;
3622                 ++txr->tx_avail;
3623
3624                 /* We clean the range if multi segment */
3625                 while (txd != eop) {
3626                         ++txd;
3627                         ++buf;
3628                         ++work;
3629                         /* wrap the ring? */
3630                         if (__predict_false(!work)) {
3631                                 work -= txr->num_desc;
3632                                 buf = txr->tx_buffers;
3633                                 txd = txr->tx_base;
3634                         }
3635                         if (buf->m_head) {
3636                                 txr->bytes +=
3637                                     buf->m_head->m_pkthdr.len;
3638                                 bus_dmamap_sync(txr->txtag,
3639                                     buf->map,
3640                                     BUS_DMASYNC_POSTWRITE);
3641                                 bus_dmamap_unload(txr->txtag,
3642                                     buf->map);
3643                                 m_freem(buf->m_head);
3644                                 buf->m_head = NULL;
3645                                 buf->map = NULL;
3646                         }
3647                         ++txr->tx_avail;
3648                         buf->eop = NULL;
3649
3650                 }
3651                 ++txr->packets;
3652                 ++processed;
3653                 ++ifp->if_opackets;
3654                 txr->watchdog_time = ticks;
3655
3656                 /* Try the next packet */
3657                 ++txd;
3658                 ++buf;
3659                 ++work;
3660                 /* reset with a wrap */
3661                 if (__predict_false(!work)) {
3662                         work -= txr->num_desc;
3663                         buf = txr->tx_buffers;
3664                         txd = txr->tx_base;
3665                 }
3666                 prefetch(txd);
3667         } while (__predict_true(--limit));
3668
3669         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3670             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3671
3672         work += txr->num_desc;
3673         txr->next_to_clean = work;
3674
3675         /*
3676         ** Watchdog calculation, we know there's
3677         ** work outstanding or the first return
3678         ** would have been taken, so none processed
3679         ** for too long indicates a hang.
3680         */
3681         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3682                 txr->queue_status = IXGBE_QUEUE_HUNG;
3683
3684         if (txr->tx_avail == txr->num_desc) {
3685                 txr->queue_status = IXGBE_QUEUE_IDLE;
3686                 return (FALSE);
3687         }
3688
3689         return TRUE;
3690 }
3691
3692 /*********************************************************************
3693  *
3694  *  Refresh mbuf buffers for RX descriptor rings
3695  *   - now keeps its own state so discards due to resource
3696  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3697  *     it just returns, keeping its placeholder, thus it can simply
3698  *     be recalled to try again.
3699  *
3700  **********************************************************************/
3701 static void
3702 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3703 {
3704         struct adapter          *adapter = rxr->adapter;
3705         bus_dma_segment_t       seg[1];
3706         struct ixgbe_rx_buf     *rxbuf;
3707         struct mbuf             *mp;
3708         int                     i, j, nsegs, error;
3709         bool                    refreshed = FALSE;
3710
3711         i = j = rxr->next_to_refresh;
3712         /* Control the loop with one beyond */
3713         if (++j == rxr->num_desc)
3714                 j = 0;
3715
3716         while (j != limit) {
3717                 rxbuf = &rxr->rx_buffers[i];
3718                 if (rxbuf->buf == NULL) {
3719                         mp = m_getjcl(M_NOWAIT, MT_DATA,
3720                             M_PKTHDR, rxr->mbuf_sz);
3721                         if (mp == NULL)
3722                                 goto update;
3723                         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3724                                 m_adj(mp, ETHER_ALIGN);
3725                 } else
3726                         mp = rxbuf->buf;
3727
3728                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3729
3730                 /* If we're dealing with an mbuf that was copied rather
3731                  * than replaced, there's no need to go through busdma.
3732                  */
3733                 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3734                         /* Get the memory mapping */
3735                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3736                             rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3737                         if (error != 0) {
3738                                 printf("Refresh mbufs: payload dmamap load"
3739                                     " failure - %d\n", error);
3740                                 m_free(mp);
3741                                 rxbuf->buf = NULL;
3742                                 goto update;
3743                         }
3744                         rxbuf->buf = mp;
3745                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3746                             BUS_DMASYNC_PREREAD);
3747                         rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3748                             htole64(seg[0].ds_addr);
3749                 } else {
3750                         rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3751                         rxbuf->flags &= ~IXGBE_RX_COPY;
3752                 }
3753
3754                 refreshed = TRUE;
3755                 /* Next is precalculated */
3756                 i = j;
3757                 rxr->next_to_refresh = i;
3758                 if (++j == rxr->num_desc)
3759                         j = 0;
3760         }
3761 update:
3762         if (refreshed) /* Update hardware tail index */
3763                 IXGBE_WRITE_REG(&adapter->hw,
3764                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3765         return;
3766 }
3767
3768 /*********************************************************************
3769  *
3770  *  Allocate memory for rx_buffer structures. Since we use one
3771  *  rx_buffer per received packet, the maximum number of rx_buffer's
3772  *  that we'll need is equal to the number of receive descriptors
3773  *  that we've allocated.
3774  *
3775  **********************************************************************/
3776 static int
3777 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3778 {
3779         struct  adapter         *adapter = rxr->adapter;
3780         device_t                dev = adapter->dev;
3781         struct ixgbe_rx_buf     *rxbuf;
3782         int                     i, bsize, error;
3783
3784         bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3785         if (!(rxr->rx_buffers =
3786             (struct ixgbe_rx_buf *) malloc(bsize,
3787             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3788                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3789                 error = ENOMEM;
3790                 goto fail;
3791         }
3792
3793         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3794                                    1, 0,        /* alignment, bounds */
3795                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3796                                    BUS_SPACE_MAXADDR,   /* highaddr */
3797                                    NULL, NULL,          /* filter, filterarg */
3798                                    MJUM16BYTES,         /* maxsize */
3799                                    1,                   /* nsegments */
3800                                    MJUM16BYTES,         /* maxsegsize */
3801                                    0,                   /* flags */
3802                                    NULL,                /* lockfunc */
3803                                    NULL,                /* lockfuncarg */
3804                                    &rxr->ptag))) {
3805                 device_printf(dev, "Unable to create RX DMA tag\n");
3806                 goto fail;
3807         }
3808
3809         for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3810                 rxbuf = &rxr->rx_buffers[i];
3811                 error = bus_dmamap_create(rxr->ptag,
3812                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3813                 if (error) {
3814                         device_printf(dev, "Unable to create RX dma map\n");
3815                         goto fail;
3816                 }
3817         }
3818
3819         return (0);
3820
3821 fail:
3822         /* Frees all, but can handle partial completion */
3823         ixgbe_free_receive_structures(adapter);
3824         return (error);
3825 }
3826
3827 /*
3828 ** Used to detect a descriptor that has
3829 ** been merged by Hardware RSC.
3830 */
3831 static inline u32
3832 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3833 {
3834         return (le32toh(rx->wb.lower.lo_dword.data) &
3835             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3836 }
3837
3838 /*********************************************************************
3839  *
3840  *  Initialize Hardware RSC (LRO) feature on 82599
3841  *  for an RX ring, this is toggled by the LRO capability
3842  *  even though it is transparent to the stack.
3843  *
3844  *  NOTE: since this HW feature only works with IPV4 and
3845  *        our testing has shown soft LRO to be as effective
3846  *        I have decided to disable this by default.
3847  *
3848  **********************************************************************/
3849 static void
3850 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3851 {
3852         struct  adapter         *adapter = rxr->adapter;
3853         struct  ixgbe_hw        *hw = &adapter->hw;
3854         u32                     rscctrl, rdrxctl;
3855
3856         /* If turning LRO/RSC off we need to disable it */
3857         if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3858                 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3859                 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3860                 return;
3861         }
3862
3863         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3864         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3865 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3866         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3867 #endif /* DEV_NETMAP */
3868         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3869         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3870         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3871
3872         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3873         rscctrl |= IXGBE_RSCCTL_RSCEN;
3874         /*
3875         ** Limit the total number of descriptors that
3876         ** can be combined, so it does not exceed 64K
3877         */
3878         if (rxr->mbuf_sz == MCLBYTES)
3879                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3880         else if (rxr->mbuf_sz == MJUMPAGESIZE)
3881                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3882         else if (rxr->mbuf_sz == MJUM9BYTES)
3883                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3884         else  /* Using 16K cluster */
3885                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3886
3887         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3888
3889         /* Enable TCP header recognition */
3890         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3891             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3892             IXGBE_PSRTYPE_TCPHDR));
3893
3894         /* Disable RSC for ACK packets */
3895         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3896             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3897
3898         rxr->hw_rsc = TRUE;
3899 }
3900
3901
3902 static void
3903 ixgbe_free_receive_ring(struct rx_ring *rxr)
3904 {
3905         struct ixgbe_rx_buf       *rxbuf;
3906         int i;
3907
3908         for (i = 0; i < rxr->num_desc; i++) {
3909                 rxbuf = &rxr->rx_buffers[i];
3910                 if (rxbuf->buf != NULL) {
3911                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3912                             BUS_DMASYNC_POSTREAD);
3913                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3914                         rxbuf->buf->m_flags |= M_PKTHDR;
3915                         m_freem(rxbuf->buf);
3916                         rxbuf->buf = NULL;
3917                 }
3918         }
3919 }
3920
3921
3922 /*********************************************************************
3923  *
3924  *  Initialize a receive ring and its buffers.
3925  *
3926  **********************************************************************/
3927 static int
3928 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3929 {
3930         struct  adapter         *adapter;
3931         struct ifnet            *ifp;
3932         device_t                dev;
3933         struct ixgbe_rx_buf     *rxbuf;
3934         bus_dma_segment_t       seg[1];
3935         struct lro_ctrl         *lro = &rxr->lro;
3936         int                     rsize, nsegs, error = 0;
3937 #ifdef DEV_NETMAP
3938         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3939         struct netmap_slot *slot;
3940 #endif /* DEV_NETMAP */
3941
3942         adapter = rxr->adapter;
3943         ifp = adapter->ifp;
3944         dev = adapter->dev;
3945
3946         /* Clear the ring contents */
3947         IXGBE_RX_LOCK(rxr);
3948 #ifdef DEV_NETMAP
3949         /* same as in ixgbe_setup_transmit_ring() */
3950         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3951 #endif /* DEV_NETMAP */
3952         rsize = roundup2(adapter->num_rx_desc *
3953             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3954         bzero((void *)rxr->rx_base, rsize);
3955         /* Cache the size */
3956         rxr->mbuf_sz = adapter->rx_mbuf_sz;
3957
3958         /* Free current RX buffer structs and their mbufs */
3959         ixgbe_free_receive_ring(rxr);
3960
3961         /* Now replenish the mbufs */
3962         for (int j = 0; j != rxr->num_desc; ++j) {
3963                 struct mbuf     *mp;
3964
3965                 rxbuf = &rxr->rx_buffers[j];
3966 #ifdef DEV_NETMAP
3967                 /*
3968                  * In netmap mode, fill the map and set the buffer
3969                  * address in the NIC ring, considering the offset
3970                  * between the netmap and NIC rings (see comment in
3971                  * ixgbe_setup_transmit_ring() ). No need to allocate
3972                  * an mbuf, so end the block with a continue;
3973                  */
3974                 if (slot) {
3975                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3976                         uint64_t paddr;
3977                         void *addr;
3978
3979                         addr = PNMB(slot + sj, &paddr);
3980                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3981                         /* Update descriptor */
3982                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3983                         continue;
3984                 }
3985 #endif /* DEV_NETMAP */
3986                 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
3987                     M_PKTHDR, adapter->rx_mbuf_sz);
3988                 if (rxbuf->buf == NULL) {
3989                         error = ENOBUFS;
3990                         goto fail;
3991                 }
3992                 mp = rxbuf->buf;
3993                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3994                 /* Get the memory mapping */
3995                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3996                     rxbuf->pmap, mp, seg,
3997                     &nsegs, BUS_DMA_NOWAIT);
3998                 if (error != 0)
3999                         goto fail;
4000                 bus_dmamap_sync(rxr->ptag,
4001                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4002                 /* Update descriptor */
4003                 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4004         }
4005
4006
4007         /* Setup our descriptor indices */
4008         rxr->next_to_check = 0;
4009         rxr->next_to_refresh = 0;
4010         rxr->lro_enabled = FALSE;
4011         rxr->rx_copies = 0;
4012         rxr->rx_bytes = 0;
4013         rxr->discard = FALSE;
4014         rxr->vtag_strip = FALSE;
4015
4016         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4017             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4018
4019         /*
4020         ** Now set up the LRO interface:
4021         */
4022         if (ixgbe_rsc_enable)
4023                 ixgbe_setup_hw_rsc(rxr);
4024         else if (ifp->if_capenable & IFCAP_LRO) {
4025                 int err = tcp_lro_init(lro);
4026                 if (err) {
4027                         device_printf(dev, "LRO Initialization failed!\n");
4028                         goto fail;
4029                 }
4030                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4031                 rxr->lro_enabled = TRUE;
4032                 lro->ifp = adapter->ifp;
4033         }
4034
4035         IXGBE_RX_UNLOCK(rxr);
4036         return (0);
4037
4038 fail:
4039         ixgbe_free_receive_ring(rxr);
4040         IXGBE_RX_UNLOCK(rxr);
4041         return (error);
4042 }
4043
4044 /*********************************************************************
4045  *
4046  *  Initialize all receive rings.
4047  *
4048  **********************************************************************/
4049 static int
4050 ixgbe_setup_receive_structures(struct adapter *adapter)
4051 {
4052         struct rx_ring *rxr = adapter->rx_rings;
4053         int j;
4054
4055         for (j = 0; j < adapter->num_queues; j++, rxr++)
4056                 if (ixgbe_setup_receive_ring(rxr))
4057                         goto fail;
4058
4059         return (0);
4060 fail:
4061         /*
4062          * Free RX buffers allocated so far, we will only handle
4063          * the rings that completed, the failing case will have
4064          * cleaned up for itself. 'j' failed, so its the terminus.
4065          */
4066         for (int i = 0; i < j; ++i) {
4067                 rxr = &adapter->rx_rings[i];
4068                 ixgbe_free_receive_ring(rxr);
4069         }
4070
4071         return (ENOBUFS);
4072 }
4073
4074 /*********************************************************************
4075  *
4076  *  Setup receive registers and features.
4077  *
4078  **********************************************************************/
4079 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4080
4081 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4082
4083 static void
4084 ixgbe_initialize_receive_units(struct adapter *adapter)
4085 {
4086         struct  rx_ring *rxr = adapter->rx_rings;
4087         struct ixgbe_hw *hw = &adapter->hw;
4088         struct ifnet   *ifp = adapter->ifp;
4089         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4090         u32             reta, mrqc = 0, hlreg, random[10];
4091
4092
4093         /*
4094          * Make sure receives are disabled while
4095          * setting up the descriptor ring
4096          */
4097         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4098         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4099             rxctrl & ~IXGBE_RXCTRL_RXEN);
4100
4101         /* Enable broadcasts */
4102         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4103         fctrl |= IXGBE_FCTRL_BAM;
4104         fctrl |= IXGBE_FCTRL_DPF;
4105         fctrl |= IXGBE_FCTRL_PMCF;
4106         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4107
4108         /* Set for Jumbo Frames? */
4109         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4110         if (ifp->if_mtu > ETHERMTU)
4111                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4112         else
4113                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4114 #ifdef DEV_NETMAP
4115         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4116         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4117                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4118         else
4119                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4120 #endif /* DEV_NETMAP */
4121         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4122
4123         bufsz = (adapter->rx_mbuf_sz +
4124             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4125
4126         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4127                 u64 rdba = rxr->rxdma.dma_paddr;
4128
4129                 /* Setup the Base and Length of the Rx Descriptor Ring */
4130                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4131                                (rdba & 0x00000000ffffffffULL));
4132                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4133                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4134                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4135
4136                 /* Set up the SRRCTL register */
4137                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4138                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4139                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4140                 srrctl |= bufsz;
4141                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4142                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4143
4144                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4145                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4146                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4147
4148                 /* Set the processing limit */
4149                 rxr->process_limit = ixgbe_rx_process_limit;
4150         }
4151
4152         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4153                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4154                               IXGBE_PSRTYPE_UDPHDR |
4155                               IXGBE_PSRTYPE_IPV4HDR |
4156                               IXGBE_PSRTYPE_IPV6HDR;
4157                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4158         }
4159
4160         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4161
4162         /* Setup RSS */
4163         if (adapter->num_queues > 1) {
4164                 int i, j;
4165                 reta = 0;
4166
4167                 /* set up random bits */
4168                 arc4rand(&random, sizeof(random), 0);
4169
4170                 /* Set up the redirection table */
4171                 for (i = 0, j = 0; i < 128; i++, j++) {
4172                         if (j == adapter->num_queues) j = 0;
4173                         reta = (reta << 8) | (j * 0x11);
4174                         if ((i & 3) == 3)
4175                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4176                 }
4177
4178                 /* Now fill our hash function seeds */
4179                 for (int i = 0; i < 10; i++)
4180                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4181
4182                 /* Perform hash on these packet types */
4183                 mrqc = IXGBE_MRQC_RSSEN
4184                      | IXGBE_MRQC_RSS_FIELD_IPV4
4185                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4186                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4187                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4188                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4189                      | IXGBE_MRQC_RSS_FIELD_IPV6
4190                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4191                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4192                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4193                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4194
4195                 /* RSS and RX IPP Checksum are mutually exclusive */
4196                 rxcsum |= IXGBE_RXCSUM_PCSD;
4197         }
4198
4199         if (ifp->if_capenable & IFCAP_RXCSUM)
4200                 rxcsum |= IXGBE_RXCSUM_PCSD;
4201
4202         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4203                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4204
4205         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4206
4207         return;
4208 }
4209
4210 /*********************************************************************
4211  *
4212  *  Free all receive rings.
4213  *
4214  **********************************************************************/
4215 static void
4216 ixgbe_free_receive_structures(struct adapter *adapter)
4217 {
4218         struct rx_ring *rxr = adapter->rx_rings;
4219
4220         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4221                 struct lro_ctrl         *lro = &rxr->lro;
4222                 ixgbe_free_receive_buffers(rxr);
4223                 /* Free LRO memory */
4224                 tcp_lro_free(lro);
4225                 /* Free the ring memory as well */
4226                 ixgbe_dma_free(adapter, &rxr->rxdma);
4227         }
4228
4229         free(adapter->rx_rings, M_DEVBUF);
4230 }
4231
4232
4233 /*********************************************************************
4234  *
4235  *  Free receive ring data structures
4236  *
4237  **********************************************************************/
4238 static void
4239 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4240 {
4241         struct adapter          *adapter = rxr->adapter;
4242         struct ixgbe_rx_buf     *rxbuf;
4243
4244         INIT_DEBUGOUT("free_receive_structures: begin");
4245
4246         /* Cleanup any existing buffers */
4247         if (rxr->rx_buffers != NULL) {
4248                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4249                         rxbuf = &rxr->rx_buffers[i];
4250                         if (rxbuf->buf != NULL) {
4251                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4252                                     BUS_DMASYNC_POSTREAD);
4253                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4254                                 rxbuf->buf->m_flags |= M_PKTHDR;
4255                                 m_freem(rxbuf->buf);
4256                         }
4257                         rxbuf->buf = NULL;
4258                         if (rxbuf->pmap != NULL) {
4259                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4260                                 rxbuf->pmap = NULL;
4261                         }
4262                 }
4263                 if (rxr->rx_buffers != NULL) {
4264                         free(rxr->rx_buffers, M_DEVBUF);
4265                         rxr->rx_buffers = NULL;
4266                 }
4267         }
4268
4269         if (rxr->ptag != NULL) {
4270                 bus_dma_tag_destroy(rxr->ptag);
4271                 rxr->ptag = NULL;
4272         }
4273
4274         return;
4275 }
4276
4277 static __inline void
4278 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4279 {
4280
4281         /*
4282          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4283          * should be computed by hardware. Also it should not have VLAN tag in
4284          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4285          */
4286         if (rxr->lro_enabled &&
4287             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4288             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4289             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4290             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4291             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4292             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4293             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4294             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4295                 /*
4296                  * Send to the stack if:
4297                  **  - LRO not enabled, or
4298                  **  - no LRO resources, or
4299                  **  - lro enqueue fails
4300                  */
4301                 if (rxr->lro.lro_cnt != 0)
4302                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4303                                 return;
4304         }
4305         IXGBE_RX_UNLOCK(rxr);
4306         (*ifp->if_input)(ifp, m);
4307         IXGBE_RX_LOCK(rxr);
4308 }
4309
4310 static __inline void
4311 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4312 {
4313         struct ixgbe_rx_buf     *rbuf;
4314
4315         rbuf = &rxr->rx_buffers[i];
4316
4317         if (rbuf->fmp != NULL) {/* Partial chain ? */
4318                 rbuf->fmp->m_flags |= M_PKTHDR;
4319                 m_freem(rbuf->fmp);
4320                 rbuf->fmp = NULL;
4321         }
4322
4323         /*
4324         ** With advanced descriptors the writeback
4325         ** clobbers the buffer addrs, so its easier
4326         ** to just free the existing mbufs and take
4327         ** the normal refresh path to get new buffers
4328         ** and mapping.
4329         */
4330         if (rbuf->buf) {
4331                 m_free(rbuf->buf);
4332                 rbuf->buf = NULL;
4333         }
4334
4335         return;
4336 }
4337
4338
4339 /*********************************************************************
4340  *
4341  *  This routine executes in interrupt context. It replenishes
4342  *  the mbufs in the descriptor and sends data which has been
4343  *  dma'ed into host memory to upper layer.
4344  *
4345  *  We loop at most count times if count is > 0, or until done if
4346  *  count < 0.
4347  *
4348  *  Return TRUE for more work, FALSE for all clean.
4349  *********************************************************************/
4350 static bool
4351 ixgbe_rxeof(struct ix_queue *que)
4352 {
4353         struct adapter          *adapter = que->adapter;
4354         struct rx_ring          *rxr = que->rxr;
4355         struct ifnet            *ifp = adapter->ifp;
4356         struct lro_ctrl         *lro = &rxr->lro;
4357         struct lro_entry        *queued;
4358         int                     i, nextp, processed = 0;
4359         u32                     staterr = 0;
4360         u16                     count = rxr->process_limit;
4361         union ixgbe_adv_rx_desc *cur;
4362         struct ixgbe_rx_buf     *rbuf, *nbuf;
4363
4364         IXGBE_RX_LOCK(rxr);
4365
4366 #ifdef DEV_NETMAP
4367         if (ifp->if_capenable & IFCAP_NETMAP) {
4368                 /*
4369                  * Same as the txeof routine: only wakeup clients on intr.
4370                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4371                  * mitigation (ixgbe_rxsync() will not look for new packets
4372                  * unless NKR_PENDINTR is set).
4373                  */
4374                 struct netmap_adapter *na = NA(ifp);
4375
4376                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4377                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4378                 IXGBE_RX_UNLOCK(rxr);
4379                 IXGBE_CORE_LOCK(adapter);
4380                 selwakeuppri(&na->rx_si, PI_NET);
4381                 IXGBE_CORE_UNLOCK(adapter);
4382                 return (FALSE);
4383         }
4384 #endif /* DEV_NETMAP */
4385         for (i = rxr->next_to_check; count != 0;) {
4386                 struct mbuf     *sendmp, *mp;
4387                 u32             rsc, ptype;
4388                 u16             len;
4389                 u16             vtag = 0;
4390                 bool            eop;
4391
4392                 /* Sync the ring. */
4393                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4394                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4395
4396                 cur = &rxr->rx_base[i];
4397                 staterr = le32toh(cur->wb.upper.status_error);
4398
4399                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4400                         break;
4401                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4402                         break;
4403
4404                 count--;
4405                 sendmp = NULL;
4406                 nbuf = NULL;
4407                 rsc = 0;
4408                 cur->wb.upper.status_error = 0;
4409                 rbuf = &rxr->rx_buffers[i];
4410                 mp = rbuf->buf;
4411
4412                 len = le16toh(cur->wb.upper.length);
4413                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4414                     IXGBE_RXDADV_PKTTYPE_MASK;
4415                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4416
4417                 /* Make sure bad packets are discarded */
4418                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4419                     (rxr->discard)) {
4420                         rxr->rx_discarded++;
4421                         if (eop)
4422                                 rxr->discard = FALSE;
4423                         else
4424                                 rxr->discard = TRUE;
4425                         ixgbe_rx_discard(rxr, i);
4426                         goto next_desc;
4427                 }
4428
4429                 /*
4430                 ** On 82599 which supports a hardware
4431                 ** LRO (called HW RSC), packets need
4432                 ** not be fragmented across sequential
4433                 ** descriptors, rather the next descriptor
4434                 ** is indicated in bits of the descriptor.
4435                 ** This also means that we might proceses
4436                 ** more than one packet at a time, something
4437                 ** that has never been true before, it
4438                 ** required eliminating global chain pointers
4439                 ** in favor of what we are doing here.  -jfv
4440                 */
4441                 if (!eop) {
4442                         /*
4443                         ** Figure out the next descriptor
4444                         ** of this frame.
4445                         */
4446                         if (rxr->hw_rsc == TRUE) {
4447                                 rsc = ixgbe_rsc_count(cur);
4448                                 rxr->rsc_num += (rsc - 1);
4449                         }
4450                         if (rsc) { /* Get hardware index */
4451                                 nextp = ((staterr &
4452                                     IXGBE_RXDADV_NEXTP_MASK) >>
4453                                     IXGBE_RXDADV_NEXTP_SHIFT);
4454                         } else { /* Just sequential */
4455                                 nextp = i + 1;
4456                                 if (nextp == adapter->num_rx_desc)
4457                                         nextp = 0;
4458                         }
4459                         nbuf = &rxr->rx_buffers[nextp];
4460                         prefetch(nbuf);
4461                 }
4462                 /*
4463                 ** Rather than using the fmp/lmp global pointers
4464                 ** we now keep the head of a packet chain in the
4465                 ** buffer struct and pass this along from one
4466                 ** descriptor to the next, until we get EOP.
4467                 */
4468                 mp->m_len = len;
4469                 /*
4470                 ** See if there is a stored head
4471                 ** that determines what we are
4472                 */
4473                 sendmp = rbuf->fmp;
4474                 if (sendmp != NULL) {  /* secondary frag */
4475                         rbuf->buf = rbuf->fmp = NULL;
4476                         mp->m_flags &= ~M_PKTHDR;
4477                         sendmp->m_pkthdr.len += mp->m_len;
4478                 } else {
4479                         /*
4480                          * Optimize.  This might be a small packet,
4481                          * maybe just a TCP ACK.  Do a fast copy that
4482                          * is cache aligned into a new mbuf, and
4483                          * leave the old mbuf+cluster for re-use.
4484                          */
4485                         if (eop && len <= IXGBE_RX_COPY_LEN) {
4486                                 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4487                                 if (sendmp != NULL) {
4488                                         sendmp->m_data +=
4489                                             IXGBE_RX_COPY_ALIGN;
4490                                         ixgbe_bcopy(mp->m_data,
4491                                             sendmp->m_data, len);
4492                                         sendmp->m_len = len;
4493                                         rxr->rx_copies++;
4494                                         rbuf->flags |= IXGBE_RX_COPY;
4495                                 }
4496                         }
4497                         if (sendmp == NULL) {
4498                                 rbuf->buf = rbuf->fmp = NULL;
4499                                 sendmp = mp;
4500                         }
4501
4502                         /* first desc of a non-ps chain */
4503                         sendmp->m_flags |= M_PKTHDR;
4504                         sendmp->m_pkthdr.len = mp->m_len;
4505                 }
4506                 ++processed;
4507
4508                 /* Pass the head pointer on */
4509                 if (eop == 0) {
4510                         nbuf->fmp = sendmp;
4511                         sendmp = NULL;
4512                         mp->m_next = nbuf->buf;
4513                 } else { /* Sending this frame */
4514                         sendmp->m_pkthdr.rcvif = ifp;
4515                         ifp->if_ipackets++;
4516                         rxr->rx_packets++;
4517                         /* capture data for AIM */
4518                         rxr->bytes += sendmp->m_pkthdr.len;
4519                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4520                         /* Process vlan info */
4521                         if ((rxr->vtag_strip) &&
4522                             (staterr & IXGBE_RXD_STAT_VP))
4523                                 vtag = le16toh(cur->wb.upper.vlan);
4524                         if (vtag) {
4525                                 sendmp->m_pkthdr.ether_vtag = vtag;
4526                                 sendmp->m_flags |= M_VLANTAG;
4527                         }
4528                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4529                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4530 #if __FreeBSD_version >= 800000
4531                         sendmp->m_pkthdr.flowid = que->msix;
4532                         sendmp->m_flags |= M_FLOWID;
4533 #endif
4534                 }
4535 next_desc:
4536                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4537                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4538
4539                 /* Advance our pointers to the next descriptor. */
4540                 if (++i == rxr->num_desc)
4541                         i = 0;
4542
4543                 /* Now send to the stack or do LRO */
4544                 if (sendmp != NULL) {
4545                         rxr->next_to_check = i;
4546                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4547                         i = rxr->next_to_check;
4548                 }
4549
4550                /* Every 8 descriptors we go to refresh mbufs */
4551                 if (processed == 8) {
4552                         ixgbe_refresh_mbufs(rxr, i);
4553                         processed = 0;
4554                 }
4555         }
4556
4557         /* Refresh any remaining buf structs */
4558         if (ixgbe_rx_unrefreshed(rxr))
4559                 ixgbe_refresh_mbufs(rxr, i);
4560
4561         rxr->next_to_check = i;
4562
4563         /*
4564          * Flush any outstanding LRO work
4565          */
4566         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4567                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4568                 tcp_lro_flush(lro, queued);
4569         }
4570
4571         IXGBE_RX_UNLOCK(rxr);
4572
4573         /*
4574         ** We still have cleaning to do?
4575         ** Schedule another interrupt if so.
4576         */
4577         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4578                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4579                 return (TRUE);
4580         }
4581
4582         return (FALSE);
4583 }
4584
4585
4586 /*********************************************************************
4587  *
4588  *  Verify that the hardware indicated that the checksum is valid.
4589  *  Inform the stack about the status of checksum so that stack
4590  *  doesn't spend time verifying the checksum.
4591  *
4592  *********************************************************************/
4593 static void
4594 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4595 {
4596         u16     status = (u16) staterr;
4597         u8      errors = (u8) (staterr >> 24);
4598         bool    sctp = FALSE;
4599
4600         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4601             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4602                 sctp = TRUE;
4603
4604         if (status & IXGBE_RXD_STAT_IPCS) {
4605                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4606                         /* IP Checksum Good */
4607                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4608                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4609
4610                 } else
4611                         mp->m_pkthdr.csum_flags = 0;
4612         }
4613         if (status & IXGBE_RXD_STAT_L4CS) {
4614                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4615 #if __FreeBSD_version >= 800000
4616                 if (sctp)
4617                         type = CSUM_SCTP_VALID;
4618 #endif
4619                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4620                         mp->m_pkthdr.csum_flags |= type;
4621                         if (!sctp)
4622                                 mp->m_pkthdr.csum_data = htons(0xffff);
4623                 }
4624         }
4625         return;
4626 }
4627
4628
4629 /*
4630 ** This routine is run via an vlan config EVENT,
4631 ** it enables us to use the HW Filter table since
4632 ** we can get the vlan id. This just creates the
4633 ** entry in the soft version of the VFTA, init will
4634 ** repopulate the real table.
4635 */
4636 static void
4637 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4638 {
4639         struct adapter  *adapter = ifp->if_softc;
4640         u16             index, bit;
4641
4642         if (ifp->if_softc !=  arg)   /* Not our event */
4643                 return;
4644
4645         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4646                 return;
4647
4648         IXGBE_CORE_LOCK(adapter);
4649         index = (vtag >> 5) & 0x7F;
4650         bit = vtag & 0x1F;
4651         adapter->shadow_vfta[index] |= (1 << bit);
4652         ++adapter->num_vlans;
4653         ixgbe_init_locked(adapter);
4654         IXGBE_CORE_UNLOCK(adapter);
4655 }
4656
4657 /*
4658 ** This routine is run via an vlan
4659 ** unconfig EVENT, remove our entry
4660 ** in the soft vfta.
4661 */
4662 static void
4663 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4664 {
4665         struct adapter  *adapter = ifp->if_softc;
4666         u16             index, bit;
4667
4668         if (ifp->if_softc !=  arg)
4669                 return;
4670
4671         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4672                 return;
4673
4674         IXGBE_CORE_LOCK(adapter);
4675         index = (vtag >> 5) & 0x7F;
4676         bit = vtag & 0x1F;
4677         adapter->shadow_vfta[index] &= ~(1 << bit);
4678         --adapter->num_vlans;
4679         /* Re-init to load the changes */
4680         ixgbe_init_locked(adapter);
4681         IXGBE_CORE_UNLOCK(adapter);
4682 }
4683
4684 static void
4685 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4686 {
4687         struct ifnet    *ifp = adapter->ifp;
4688         struct ixgbe_hw *hw = &adapter->hw;
4689         struct rx_ring  *rxr;
4690         u32             ctrl;
4691
4692
4693         /*
4694         ** We get here thru init_locked, meaning
4695         ** a soft reset, this has already cleared
4696         ** the VFTA and other state, so if there
4697         ** have been no vlan's registered do nothing.
4698         */
4699         if (adapter->num_vlans == 0)
4700                 return;
4701
4702         /*
4703         ** A soft reset zero's out the VFTA, so
4704         ** we need to repopulate it now.
4705         */
4706         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4707                 if (adapter->shadow_vfta[i] != 0)
4708                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4709                             adapter->shadow_vfta[i]);
4710
4711         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4712         /* Enable the Filter Table if enabled */
4713         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4714                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4715                 ctrl |= IXGBE_VLNCTRL_VFE;
4716         }
4717         if (hw->mac.type == ixgbe_mac_82598EB)
4718                 ctrl |= IXGBE_VLNCTRL_VME;
4719         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4720
4721         /* Setup the queues for vlans */
4722         for (int i = 0; i < adapter->num_queues; i++) {
4723                 rxr = &adapter->rx_rings[i];
4724                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4725                 if (hw->mac.type != ixgbe_mac_82598EB) {
4726                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4727                         ctrl |= IXGBE_RXDCTL_VME;
4728                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4729                 }
4730                 rxr->vtag_strip = TRUE;
4731         }
4732 }
4733
4734 static void
4735 ixgbe_enable_intr(struct adapter *adapter)
4736 {
4737         struct ixgbe_hw *hw = &adapter->hw;
4738         struct ix_queue *que = adapter->queues;
4739         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4740
4741
4742         /* Enable Fan Failure detection */
4743         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4744                     mask |= IXGBE_EIMS_GPI_SDP1;
4745
4746         switch (adapter->hw.mac.type) {
4747                 case ixgbe_mac_82599EB:
4748                         mask |= IXGBE_EIMS_ECC;
4749                         mask |= IXGBE_EIMS_GPI_SDP0;
4750                         mask |= IXGBE_EIMS_GPI_SDP1;
4751                         mask |= IXGBE_EIMS_GPI_SDP2;
4752 #ifdef IXGBE_FDIR
4753                         mask |= IXGBE_EIMS_FLOW_DIR;
4754 #endif
4755                         break;
4756                 case ixgbe_mac_X540:
4757                         mask |= IXGBE_EIMS_ECC;
4758 #ifdef IXGBE_FDIR
4759                         mask |= IXGBE_EIMS_FLOW_DIR;
4760 #endif
4761                 /* falls through */
4762                 default:
4763                         break;
4764         }
4765
4766         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4767
4768         /* With RSS we use auto clear */
4769         if (adapter->msix_mem) {
4770                 mask = IXGBE_EIMS_ENABLE_MASK;
4771                 /* Don't autoclear Link */
4772                 mask &= ~IXGBE_EIMS_OTHER;
4773                 mask &= ~IXGBE_EIMS_LSC;
4774                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4775         }
4776
4777         /*
4778         ** Now enable all queues, this is done separately to
4779         ** allow for handling the extended (beyond 32) MSIX
4780         ** vectors that can be used by 82599
4781         */
4782         for (int i = 0; i < adapter->num_queues; i++, que++)
4783                 ixgbe_enable_queue(adapter, que->msix);
4784
4785         IXGBE_WRITE_FLUSH(hw);
4786
4787         return;
4788 }
4789
4790 static void
4791 ixgbe_disable_intr(struct adapter *adapter)
4792 {
4793         if (adapter->msix_mem)
4794                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4795         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4796                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4797         } else {
4798                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4799                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4800                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4801         }
4802         IXGBE_WRITE_FLUSH(&adapter->hw);
4803         return;
4804 }
4805
4806 u16
4807 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4808 {
4809         u16 value;
4810
4811         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4812             reg, 2);
4813
4814         return (value);
4815 }
4816
4817 void
4818 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4819 {
4820         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4821             reg, value, 2);
4822
4823         return;
4824 }
4825
4826 /*
4827 ** Setup the correct IVAR register for a particular MSIX interrupt
4828 **   (yes this is all very magic and confusing :)
4829 **  - entry is the register array entry
4830 **  - vector is the MSIX vector for this queue
4831 **  - type is RX/TX/MISC
4832 */
4833 static void
4834 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4835 {
4836         struct ixgbe_hw *hw = &adapter->hw;
4837         u32 ivar, index;
4838
4839         vector |= IXGBE_IVAR_ALLOC_VAL;
4840
4841         switch (hw->mac.type) {
4842
4843         case ixgbe_mac_82598EB:
4844                 if (type == -1)
4845                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4846                 else
4847                         entry += (type * 64);
4848                 index = (entry >> 2) & 0x1F;
4849                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4850                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4851                 ivar |= (vector << (8 * (entry & 0x3)));
4852                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4853                 break;
4854
4855         case ixgbe_mac_82599EB:
4856         case ixgbe_mac_X540:
4857                 if (type == -1) { /* MISC IVAR */
4858                         index = (entry & 1) * 8;
4859                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4860                         ivar &= ~(0xFF << index);
4861                         ivar |= (vector << index);
4862                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4863                 } else {        /* RX/TX IVARS */
4864                         index = (16 * (entry & 1)) + (8 * type);
4865                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4866                         ivar &= ~(0xFF << index);
4867                         ivar |= (vector << index);
4868                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4869                 }
4870
4871         default:
4872                 break;
4873         }
4874 }
4875
4876 static void
4877 ixgbe_configure_ivars(struct adapter *adapter)
4878 {
4879         struct  ix_queue *que = adapter->queues;
4880         u32 newitr;
4881
4882         if (ixgbe_max_interrupt_rate > 0)
4883                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
4884         else
4885                 newitr = 0;
4886
4887         for (int i = 0; i < adapter->num_queues; i++, que++) {
4888                 /* First the RX queue entry */
4889                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4890                 /* ... and the TX */
4891                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4892                 /* Set an Initial EITR value */
4893                 IXGBE_WRITE_REG(&adapter->hw,
4894                     IXGBE_EITR(que->msix), newitr);
4895         }
4896
4897         /* For the Link interrupt */
4898         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4899 }
4900
4901 /*
4902 ** ixgbe_sfp_probe - called in the local timer to
4903 ** determine if a port had optics inserted.
4904 */
4905 static bool ixgbe_sfp_probe(struct adapter *adapter)
4906 {
4907         struct ixgbe_hw *hw = &adapter->hw;
4908         device_t        dev = adapter->dev;
4909         bool            result = FALSE;
4910
4911         if ((hw->phy.type == ixgbe_phy_nl) &&
4912             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4913                 s32 ret = hw->phy.ops.identify_sfp(hw);
4914                 if (ret)
4915                         goto out;
4916                 ret = hw->phy.ops.reset(hw);
4917                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4918                         device_printf(dev,"Unsupported SFP+ module detected!");
4919                         printf(" Reload driver with supported module.\n");
4920                         adapter->sfp_probe = FALSE;
4921                         goto out;
4922                 } else
4923                         device_printf(dev,"SFP+ module detected!\n");
4924                 /* We now have supported optics */
4925                 adapter->sfp_probe = FALSE;
4926                 /* Set the optics type so system reports correctly */
4927                 ixgbe_setup_optics(adapter);
4928                 result = TRUE;
4929         }
4930 out:
4931         return (result);
4932 }
4933
4934 /*
4935 ** Tasklet handler for MSIX Link interrupts
4936 **  - do outside interrupt since it might sleep
4937 */
4938 static void
4939 ixgbe_handle_link(void *context, int pending)
4940 {
4941         struct adapter  *adapter = context;
4942
4943         ixgbe_check_link(&adapter->hw,
4944             &adapter->link_speed, &adapter->link_up, 0);
4945         ixgbe_update_link_status(adapter);
4946 }
4947
4948 /*
4949 ** Tasklet for handling SFP module interrupts
4950 */
4951 static void
4952 ixgbe_handle_mod(void *context, int pending)
4953 {
4954         struct adapter  *adapter = context;
4955         struct ixgbe_hw *hw = &adapter->hw;
4956         device_t        dev = adapter->dev;
4957         u32 err;
4958
4959         err = hw->phy.ops.identify_sfp(hw);
4960         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4961                 device_printf(dev,
4962                     "Unsupported SFP+ module type was detected.\n");
4963                 return;
4964         }
4965         err = hw->mac.ops.setup_sfp(hw);
4966         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4967                 device_printf(dev,
4968                     "Setup failure - unsupported SFP+ module type.\n");
4969                 return;
4970         }
4971         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4972         return;
4973 }
4974
4975
4976 /*
4977 ** Tasklet for handling MSF (multispeed fiber) interrupts
4978 */
4979 static void
4980 ixgbe_handle_msf(void *context, int pending)
4981 {
4982         struct adapter  *adapter = context;
4983         struct ixgbe_hw *hw = &adapter->hw;
4984         u32 autoneg;
4985         bool negotiate;
4986
4987         autoneg = hw->phy.autoneg_advertised;
4988         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4989                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4990         if (hw->mac.ops.setup_link)
4991                 hw->mac.ops.setup_link(hw, autoneg, TRUE);
4992         return;
4993 }
4994
4995 #ifdef IXGBE_FDIR
4996 /*
4997 ** Tasklet for reinitializing the Flow Director filter table
4998 */
4999 static void
5000 ixgbe_reinit_fdir(void *context, int pending)
5001 {
5002         struct adapter  *adapter = context;
5003         struct ifnet   *ifp = adapter->ifp;
5004
5005         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5006                 return;
5007         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5008         adapter->fdir_reinit = 0;
5009         /* re-enable flow director interrupts */
5010         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5011         /* Restart the interface */
5012         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5013         return;
5014 }
5015 #endif
5016
5017 /**********************************************************************
5018  *
5019  *  Update the board statistics counters.
5020  *
5021  **********************************************************************/
5022 static void
5023 ixgbe_update_stats_counters(struct adapter *adapter)
5024 {
5025         struct ifnet   *ifp = adapter->ifp;
5026         struct ixgbe_hw *hw = &adapter->hw;
5027         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5028         u64  total_missed_rx = 0;
5029
5030         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5031         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5032         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5033         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5034
5035         /*
5036         ** Note: these are for the 8 possible traffic classes,
5037         **       which in current implementation is unused,
5038         **       therefore only 0 should read real data.
5039         */
5040         for (int i = 0; i < 8; i++) {
5041                 u32 mp;
5042                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5043                 /* missed_rx tallies misses for the gprc workaround */
5044                 missed_rx += mp;
5045                 /* global total per queue */
5046                 adapter->stats.mpc[i] += mp;
5047                 /* Running comprehensive total for stats display */
5048                 total_missed_rx += adapter->stats.mpc[i];
5049                 if (hw->mac.type == ixgbe_mac_82598EB) {
5050                         adapter->stats.rnbc[i] +=
5051                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5052                         adapter->stats.qbtc[i] +=
5053                             IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5054                         adapter->stats.qbrc[i] +=
5055                             IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5056                         adapter->stats.pxonrxc[i] +=
5057                             IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5058                 } else
5059                         adapter->stats.pxonrxc[i] +=
5060                             IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5061                 adapter->stats.pxontxc[i] +=
5062                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5063                 adapter->stats.pxofftxc[i] +=
5064                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5065                 adapter->stats.pxoffrxc[i] +=
5066                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5067                 adapter->stats.pxon2offc[i] +=
5068                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5069         }
5070         for (int i = 0; i < 16; i++) {
5071                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5072                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5073                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5074         }
5075         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5076         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5077         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5078
5079         /* Hardware workaround, gprc counts missed packets */
5080         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5081         adapter->stats.gprc -= missed_rx;
5082
5083         if (hw->mac.type != ixgbe_mac_82598EB) {
5084                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5085                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5086                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5087                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5088                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5089                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5090                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5091                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5092         } else {
5093                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5094                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5095                 /* 82598 only has a counter in the high register */
5096                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5097                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5098                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5099         }
5100
5101         /*
5102          * Workaround: mprc hardware is incorrectly counting
5103          * broadcasts, so for now we subtract those.
5104          */
5105         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5106         adapter->stats.bprc += bprc;
5107         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5108         if (hw->mac.type == ixgbe_mac_82598EB)
5109                 adapter->stats.mprc -= bprc;
5110
5111         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5112         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5113         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5114         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5115         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5116         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5117
5118         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5119         adapter->stats.lxontxc += lxon;
5120         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5121         adapter->stats.lxofftxc += lxoff;
5122         total = lxon + lxoff;
5123
5124         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5125         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5126         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5127         adapter->stats.gptc -= total;
5128         adapter->stats.mptc -= total;
5129         adapter->stats.ptc64 -= total;
5130         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5131
5132         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5133         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5134         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5135         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5136         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5137         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5138         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5139         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5140         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5141         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5142         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5143         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5144         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5145         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5146         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5147         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5148         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5149         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5150         /* Only read FCOE on 82599 */
5151         if (hw->mac.type != ixgbe_mac_82598EB) {
5152                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5153                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5154                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5155                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5156                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5157         }
5158
5159         /* Fill out the OS statistics structure */
5160         ifp->if_ipackets = adapter->stats.gprc;
5161         ifp->if_opackets = adapter->stats.gptc;
5162         ifp->if_ibytes = adapter->stats.gorc;
5163         ifp->if_obytes = adapter->stats.gotc;
5164         ifp->if_imcasts = adapter->stats.mprc;
5165         ifp->if_omcasts = adapter->stats.mptc;
5166         ifp->if_collisions = 0;
5167
5168         /* Rx Errors */
5169         ifp->if_iqdrops = total_missed_rx;
5170         ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec;
5171 }
5172
5173 /** ixgbe_sysctl_tdh_handler - Handler function
5174  *  Retrieves the TDH value from the hardware
5175  */
5176 static int
5177 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5178 {
5179         int error;
5180
5181         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5182         if (!txr) return 0;
5183
5184         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5185         error = sysctl_handle_int(oidp, &val, 0, req);
5186         if (error || !req->newptr)
5187                 return error;
5188         return 0;
5189 }
5190
5191 /** ixgbe_sysctl_tdt_handler - Handler function
5192  *  Retrieves the TDT value from the hardware
5193  */
5194 static int
5195 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5196 {
5197         int error;
5198
5199         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5200         if (!txr) return 0;
5201
5202         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5203         error = sysctl_handle_int(oidp, &val, 0, req);
5204         if (error || !req->newptr)
5205                 return error;
5206         return 0;
5207 }
5208
5209 /** ixgbe_sysctl_rdh_handler - Handler function
5210  *  Retrieves the RDH value from the hardware
5211  */
5212 static int
5213 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5214 {
5215         int error;
5216
5217         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5218         if (!rxr) return 0;
5219
5220         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5221         error = sysctl_handle_int(oidp, &val, 0, req);
5222         if (error || !req->newptr)
5223                 return error;
5224         return 0;
5225 }
5226
5227 /** ixgbe_sysctl_rdt_handler - Handler function
5228  *  Retrieves the RDT value from the hardware
5229  */
5230 static int
5231 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5232 {
5233         int error;
5234
5235         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5236         if (!rxr) return 0;
5237
5238         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5239         error = sysctl_handle_int(oidp, &val, 0, req);
5240         if (error || !req->newptr)
5241                 return error;
5242         return 0;
5243 }
5244
5245 static int
5246 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5247 {
5248         int error;
5249         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5250         unsigned int reg, usec, rate;
5251
5252         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5253         usec = ((reg & 0x0FF8) >> 3);
5254         if (usec > 0)
5255                 rate = 500000 / usec;
5256         else
5257                 rate = 0;
5258         error = sysctl_handle_int(oidp, &rate, 0, req);
5259         if (error || !req->newptr)
5260                 return error;
5261         reg &= ~0xfff; /* default, no limitation */
5262         ixgbe_max_interrupt_rate = 0;
5263         if (rate > 0 && rate < 500000) {
5264                 if (rate < 1000)
5265                         rate = 1000;
5266                 ixgbe_max_interrupt_rate = rate;
5267                 reg |= ((4000000/rate) & 0xff8 );
5268         }
5269         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5270         return 0;
5271 }
5272
5273 /*
5274  * Add sysctl variables, one per statistic, to the system.
5275  */
5276 static void
5277 ixgbe_add_hw_stats(struct adapter *adapter)
5278 {
5279
5280         device_t dev = adapter->dev;
5281
5282         struct tx_ring *txr = adapter->tx_rings;
5283         struct rx_ring *rxr = adapter->rx_rings;
5284
5285         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5286         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5287         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5288         struct ixgbe_hw_stats *stats = &adapter->stats;
5289
5290         struct sysctl_oid *stat_node, *queue_node;
5291         struct sysctl_oid_list *stat_list, *queue_list;
5292
5293 #define QUEUE_NAME_LEN 32
5294         char namebuf[QUEUE_NAME_LEN];
5295
5296         /* Driver Statistics */
5297         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5298                         CTLFLAG_RD, &adapter->dropped_pkts,
5299                         "Driver dropped packets");
5300         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5301                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5302                         "m_defrag() failed");
5303         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5304                         CTLFLAG_RD, &adapter->watchdog_events,
5305                         "Watchdog timeouts");
5306         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5307                         CTLFLAG_RD, &adapter->link_irq,
5308                         "Link MSIX IRQ Handled");
5309
5310         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5311                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5312                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5313                                             CTLFLAG_RD, NULL, "Queue Name");
5314                 queue_list = SYSCTL_CHILDREN(queue_node);
5315
5316                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5317                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5318                                 sizeof(&adapter->queues[i]),
5319                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5320                                 "Interrupt Rate");
5321                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5322                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
5323                                 "irqs on this queue");
5324                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5325                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5326                                 ixgbe_sysctl_tdh_handler, "IU",
5327                                 "Transmit Descriptor Head");
5328                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5329                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5330                                 ixgbe_sysctl_tdt_handler, "IU",
5331                                 "Transmit Descriptor Tail");
5332                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5333                                 CTLFLAG_RD, &txr->tso_tx,
5334                                 "TSO");
5335                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5336                                 CTLFLAG_RD, &txr->no_tx_dma_setup,
5337                                 "Driver tx dma failure in xmit");
5338                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5339                                 CTLFLAG_RD, &txr->no_desc_avail,
5340                                 "Queue No Descriptor Available");
5341                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5342                                 CTLFLAG_RD, &txr->total_packets,
5343                                 "Queue Packets Transmitted");
5344         }
5345
5346         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5347                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5348                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5349                                             CTLFLAG_RD, NULL, "Queue Name");
5350                 queue_list = SYSCTL_CHILDREN(queue_node);
5351
5352                 struct lro_ctrl *lro = &rxr->lro;
5353
5354                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5355                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5356                                             CTLFLAG_RD, NULL, "Queue Name");
5357                 queue_list = SYSCTL_CHILDREN(queue_node);
5358
5359                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5360                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5361                                 ixgbe_sysctl_rdh_handler, "IU",
5362                                 "Receive Descriptor Head");
5363                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5364                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5365                                 ixgbe_sysctl_rdt_handler, "IU",
5366                                 "Receive Descriptor Tail");
5367                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5368                                 CTLFLAG_RD, &rxr->rx_packets,
5369                                 "Queue Packets Received");
5370                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5371                                 CTLFLAG_RD, &rxr->rx_bytes,
5372                                 "Queue Bytes Received");
5373                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5374                                 CTLFLAG_RD, &rxr->rx_copies,
5375                                 "Copied RX Frames");
5376                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5377                                 CTLFLAG_RD, &lro->lro_queued, 0,
5378                                 "LRO Queued");
5379                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5380                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5381                                 "LRO Flushed");
5382         }
5383
5384         /* MAC stats get the own sub node */
5385
5386         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5387                                     CTLFLAG_RD, NULL, "MAC Statistics");
5388         stat_list = SYSCTL_CHILDREN(stat_node);
5389
5390         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5391                         CTLFLAG_RD, &stats->crcerrs,
5392                         "CRC Errors");
5393         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5394                         CTLFLAG_RD, &stats->illerrc,
5395                         "Illegal Byte Errors");
5396         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5397                         CTLFLAG_RD, &stats->errbc,
5398                         "Byte Errors");
5399         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5400                         CTLFLAG_RD, &stats->mspdc,
5401                         "MAC Short Packets Discarded");
5402         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5403                         CTLFLAG_RD, &stats->mlfc,
5404                         "MAC Local Faults");
5405         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5406                         CTLFLAG_RD, &stats->mrfc,
5407                         "MAC Remote Faults");
5408         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5409                         CTLFLAG_RD, &stats->rlec,
5410                         "Receive Length Errors");
5411
5412         /* Flow Control stats */
5413         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5414                         CTLFLAG_RD, &stats->lxontxc,
5415                         "Link XON Transmitted");
5416         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5417                         CTLFLAG_RD, &stats->lxonrxc,
5418                         "Link XON Received");
5419         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5420                         CTLFLAG_RD, &stats->lxofftxc,
5421                         "Link XOFF Transmitted");
5422         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5423                         CTLFLAG_RD, &stats->lxoffrxc,
5424                         "Link XOFF Received");
5425
5426         /* Packet Reception Stats */
5427         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5428                         CTLFLAG_RD, &stats->tor,
5429                         "Total Octets Received");
5430         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5431                         CTLFLAG_RD, &stats->gorc,
5432                         "Good Octets Received");
5433         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5434                         CTLFLAG_RD, &stats->tpr,
5435                         "Total Packets Received");
5436         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5437                         CTLFLAG_RD, &stats->gprc,
5438                         "Good Packets Received");
5439         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5440                         CTLFLAG_RD, &stats->mprc,
5441                         "Multicast Packets Received");
5442         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5443                         CTLFLAG_RD, &stats->bprc,
5444                         "Broadcast Packets Received");
5445         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5446                         CTLFLAG_RD, &stats->prc64,
5447                         "64 byte frames received ");
5448         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5449                         CTLFLAG_RD, &stats->prc127,
5450                         "65-127 byte frames received");
5451         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5452                         CTLFLAG_RD, &stats->prc255,
5453                         "128-255 byte frames received");
5454         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5455                         CTLFLAG_RD, &stats->prc511,
5456                         "256-511 byte frames received");
5457         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5458                         CTLFLAG_RD, &stats->prc1023,
5459                         "512-1023 byte frames received");
5460         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5461                         CTLFLAG_RD, &stats->prc1522,
5462                         "1023-1522 byte frames received");
5463         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5464                         CTLFLAG_RD, &stats->ruc,
5465                         "Receive Undersized");
5466         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5467                         CTLFLAG_RD, &stats->rfc,
5468                         "Fragmented Packets Received ");
5469         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5470                         CTLFLAG_RD, &stats->roc,
5471                         "Oversized Packets Received");
5472         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5473                         CTLFLAG_RD, &stats->rjc,
5474                         "Received Jabber");
5475         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5476                         CTLFLAG_RD, &stats->mngprc,
5477                         "Management Packets Received");
5478         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5479                         CTLFLAG_RD, &stats->mngptc,
5480                         "Management Packets Dropped");
5481         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5482                         CTLFLAG_RD, &stats->xec,
5483                         "Checksum Errors");
5484
5485         /* Packet Transmission Stats */
5486         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5487                         CTLFLAG_RD, &stats->gotc,
5488                         "Good Octets Transmitted");
5489         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5490                         CTLFLAG_RD, &stats->tpt,
5491                         "Total Packets Transmitted");
5492         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5493                         CTLFLAG_RD, &stats->gptc,
5494                         "Good Packets Transmitted");
5495         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5496                         CTLFLAG_RD, &stats->bptc,
5497                         "Broadcast Packets Transmitted");
5498         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5499                         CTLFLAG_RD, &stats->mptc,
5500                         "Multicast Packets Transmitted");
5501         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5502                         CTLFLAG_RD, &stats->mngptc,
5503                         "Management Packets Transmitted");
5504         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5505                         CTLFLAG_RD, &stats->ptc64,
5506                         "64 byte frames transmitted ");
5507         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5508                         CTLFLAG_RD, &stats->ptc127,
5509                         "65-127 byte frames transmitted");
5510         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5511                         CTLFLAG_RD, &stats->ptc255,
5512                         "128-255 byte frames transmitted");
5513         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5514                         CTLFLAG_RD, &stats->ptc511,
5515                         "256-511 byte frames transmitted");
5516         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5517                         CTLFLAG_RD, &stats->ptc1023,
5518                         "512-1023 byte frames transmitted");
5519         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5520                         CTLFLAG_RD, &stats->ptc1522,
5521                         "1024-1522 byte frames transmitted");
5522 }
5523
5524 /*
5525 ** Set flow control using sysctl:
5526 ** Flow control values:
5527 **      0 - off
5528 **      1 - rx pause
5529 **      2 - tx pause
5530 **      3 - full
5531 */
5532 static int
5533 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5534 {
5535         int error, last;
5536         struct adapter *adapter = (struct adapter *) arg1;
5537
5538         last = adapter->fc;
5539         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5540         if ((error) || (req->newptr == NULL))
5541                 return (error);
5542
5543         /* Don't bother if it's not changed */
5544         if (adapter->fc == last)
5545                 return (0);
5546
5547         switch (adapter->fc) {
5548                 case ixgbe_fc_rx_pause:
5549                 case ixgbe_fc_tx_pause:
5550                 case ixgbe_fc_full:
5551                         adapter->hw.fc.requested_mode = adapter->fc;
5552                         if (adapter->num_queues > 1)
5553                                 ixgbe_disable_rx_drop(adapter);
5554                         break;
5555                 case ixgbe_fc_none:
5556                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5557                         if (adapter->num_queues > 1)
5558                                 ixgbe_enable_rx_drop(adapter);
5559                         break;
5560                 default:
5561                         adapter->fc = last;
5562                         return (EINVAL);
5563         }
5564         /* Don't autoneg if forcing a value */
5565         adapter->hw.fc.disable_fc_autoneg = TRUE;
5566         ixgbe_fc_enable(&adapter->hw);
5567         return error;
5568 }
5569
5570 /*
5571 ** Control link advertise speed:
5572 **      1 - advertise only 1G
5573 **      2 - advertise 100Mb
5574 **      3 - advertise normal
5575 */
5576 static int
5577 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5578 {
5579         int                     error = 0;
5580         struct adapter          *adapter;
5581         device_t                dev;
5582         struct ixgbe_hw         *hw;
5583         ixgbe_link_speed        speed, last;
5584
5585         adapter = (struct adapter *) arg1;
5586         dev = adapter->dev;
5587         hw = &adapter->hw;
5588         last = adapter->advertise;
5589
5590         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5591         if ((error) || (req->newptr == NULL))
5592                 return (error);
5593
5594         if (adapter->advertise == last) /* no change */
5595                 return (0);
5596
5597         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5598             (hw->phy.multispeed_fiber)))
5599                 return (EINVAL);
5600
5601         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5602                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5603                 return (EINVAL);
5604         }
5605
5606         if (adapter->advertise == 1)
5607                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5608         else if (adapter->advertise == 2)
5609                 speed = IXGBE_LINK_SPEED_100_FULL;
5610         else if (adapter->advertise == 3)
5611                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5612                         IXGBE_LINK_SPEED_10GB_FULL;
5613         else {  /* bogus value */
5614                 adapter->advertise = last;
5615                 return (EINVAL);
5616         }
5617
5618         hw->mac.autotry_restart = TRUE;
5619         hw->mac.ops.setup_link(hw, speed, TRUE);
5620
5621         return (error);
5622 }
5623
5624 /*
5625 ** Thermal Shutdown Trigger
5626 **   - cause a Thermal Overtemp IRQ
5627 */
5628 static int
5629 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5630 {
5631         int             error, fire = 0;
5632         struct adapter  *adapter = (struct adapter *) arg1;
5633         struct ixgbe_hw *hw = &adapter->hw;
5634
5635
5636         if (hw->mac.type != ixgbe_mac_X540)
5637                 return (0);
5638
5639         error = sysctl_handle_int(oidp, &fire, 0, req);
5640         if ((error) || (req->newptr == NULL))
5641                 return (error);
5642
5643         if (fire) {
5644                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5645                 reg |= IXGBE_EICR_TS;
5646                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5647         }
5648
5649         return (0);
5650 }
5651
5652 /*
5653 ** Enable the hardware to drop packets when the buffer is
5654 ** full. This is useful when multiqueue,so that no single
5655 ** queue being full stalls the entire RX engine. We only
5656 ** enable this when Multiqueue AND when Flow Control is
5657 ** disabled.
5658 */
5659 static void
5660 ixgbe_enable_rx_drop(struct adapter *adapter)
5661 {
5662         struct ixgbe_hw *hw = &adapter->hw;
5663
5664         for (int i = 0; i < adapter->num_queues; i++) {
5665                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5666                 srrctl |= IXGBE_SRRCTL_DROP_EN;
5667                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5668         }
5669 }
5670
5671 static void
5672 ixgbe_disable_rx_drop(struct adapter *adapter)
5673 {
5674         struct ixgbe_hw *hw = &adapter->hw;
5675
5676         for (int i = 0; i < adapter->num_queues; i++) {
5677                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5678                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5679                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5680         }
5681 }